1 diff --git a/Makefile.am b/Makefile.am
2 index a0431a6..dc594f8 100644
5 @@ -91,11 +91,14 @@ EXTRA_DIST += \
12 + src/loc/country-list.h \
16 + src/loc/network-list.h \
18 src/loc/stringpool.h \
20 @@ -107,9 +110,12 @@ lib_LTLIBRARIES = \
21 src_libloc_la_SOURCES = \
26 + src/country-list.c \
29 + src/network-list.c \
33 @@ -312,6 +318,7 @@ check_PROGRAMS = \
37 + src/test-network-list \
41 @@ -351,6 +358,15 @@ src_test_network_CFLAGS = \
42 src_test_network_LDADD = \
45 +src_test_network_list_SOURCES = \
46 + src/test-network-list.c
48 +src_test_network_list_CFLAGS = \
51 +src_test_network_list_LDADD = \
54 src_test_stringpool_SOURCES = \
57 @@ -390,7 +406,7 @@ MANPAGES_XML = $(patsubst %.txt,%.xml,$(MANPAGES_TXT))
59 man: $(MANPAGES) $(MANPAGES_HTML)
66 diff --git a/configure.ac b/configure.ac
67 index 2364dfd..9eb9012 100644
75 [location@lists.ipfire.org],
77 [https://location.ipfire.org/])
78 @@ -43,16 +43,16 @@ AC_PROG_MKDIR_P
80 # - man ------------------------------------------------------------------------
83 -AC_ARG_ENABLE(manpages, AS_HELP_STRING([--disable-man-pages],
85 +AC_ARG_ENABLE(man_pages, AS_HELP_STRING([--disable-man-pages],
86 [do not install man pages]))
87 -AS_IF([test "x$enable_manpages" != xno], [have_manpages=yes])
88 -AM_CONDITIONAL(ENABLE_MANPAGES, [test "x$have_manpages" = "xyes"])
89 +AS_IF([test "x$enable_man_pages" != xno], [have_man_pages=yes])
90 +AM_CONDITIONAL(ENABLE_MAN_PAGES, [test "x$have_man_pages" = "xyes"])
92 AC_PATH_PROG([XSLTPROC], [xsltproc])
94 AC_CHECK_PROGS(ASCIIDOC, [asciidoc])
95 -if test "${have_manpages}" = "yes" && test -z "${ASCIIDOC}"; then
96 +if test "${have_man_pages}" = "yes" && test -z "${ASCIIDOC}"; then
97 AC_MSG_ERROR([Required program 'asciidoc' not found])
99 # - debug ----------------------------------------------------------------------
100 diff --git a/src/.gitignore b/src/.gitignore
101 index caf80b5..3ccbdb8 100644
104 @@ -10,5 +10,6 @@ test-libloc
111 diff --git a/src/as-list.c b/src/as-list.c
113 index 0000000..5acbb8a
118 + libloc - A library to determine the location of someone on the Internet
120 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
122 + This library is free software; you can redistribute it and/or
123 + modify it under the terms of the GNU Lesser General Public
124 + License as published by the Free Software Foundation; either
125 + version 2.1 of the License, or (at your option) any later version.
127 + This library is distributed in the hope that it will be useful,
128 + but WITHOUT ANY WARRANTY; without even the implied warranty of
129 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
130 + Lesser General Public License for more details.
137 +#include <loc/as-list.h>
138 +#include <loc/private.h>
140 +struct loc_as_list {
141 + struct loc_ctx* ctx;
144 + struct loc_as** elements;
145 + size_t elements_size;
150 +static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
151 + DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
152 + list, size, list->elements_size + size);
154 + struct loc_as** elements = reallocarray(list->elements,
155 + list->elements_size + size, sizeof(*list->elements));
159 + list->elements = elements;
160 + list->elements_size += size;
165 +LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
166 + struct loc_as_list** list) {
167 + struct loc_as_list* l = calloc(1, sizeof(*l));
171 + l->ctx = loc_ref(ctx);
174 + DEBUG(l->ctx, "AS list allocated at %p\n", l);
180 +LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
186 +static void loc_as_list_free(struct loc_as_list* list) {
187 + DEBUG(list->ctx, "Releasing AS list at %p\n", list);
189 + loc_as_list_clear(list);
191 + loc_unref(list->ctx);
195 +LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
199 + if (--list->refcount > 0)
202 + loc_as_list_free(list);
206 +LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
210 +LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
211 + return list->size == 0;
214 +LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
215 + if (!list->elements)
218 + for (unsigned int i = 0; i < list->size; i++)
219 + loc_as_unref(list->elements[i]);
221 + free(list->elements);
222 + list->elements = NULL;
223 + list->elements_size = 0;
228 +LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
230 + if (index >= list->size)
233 + return loc_as_ref(list->elements[index]);
236 +LOC_EXPORT int loc_as_list_append(
237 + struct loc_as_list* list, struct loc_as* as) {
238 + if (loc_as_list_contains(list, as))
241 + // Check if we have space left
242 + if (list->size >= list->elements_size) {
243 + int r = loc_as_list_grow(list, 64);
248 + DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
250 + list->elements[list->size++] = loc_as_ref(as);
255 +LOC_EXPORT int loc_as_list_contains(
256 + struct loc_as_list* list, struct loc_as* as) {
257 + for (unsigned int i = 0; i < list->size; i++) {
258 + if (loc_as_cmp(as, list->elements[i]) == 0)
265 +LOC_EXPORT int loc_as_list_contains_number(
266 + struct loc_as_list* list, uint32_t number) {
269 + int r = loc_as_new(list->ctx, &as, number);
273 + r = loc_as_list_contains(list, as);
278 diff --git a/src/as.c b/src/as.c
279 index e1fbb01..757bf3d 100644
282 @@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
285 LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) {
286 - as->name = strdup(name);
291 + as->name = strdup(name);
297 @@ -139,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
301 + // Cannot match anything when name is not set
305 // Search if string is in name
306 if (strcasestr(as->name, string) != NULL)
308 diff --git a/src/country-list.c b/src/country-list.c
310 index 0000000..cc36740
312 +++ b/src/country-list.c
315 + libloc - A library to determine the location of someone on the Internet
317 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
319 + This library is free software; you can redistribute it and/or
320 + modify it under the terms of the GNU Lesser General Public
321 + License as published by the Free Software Foundation; either
322 + version 2.1 of the License, or (at your option) any later version.
324 + This library is distributed in the hope that it will be useful,
325 + but WITHOUT ANY WARRANTY; without even the implied warranty of
326 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
327 + Lesser General Public License for more details.
333 +#include <loc/country.h>
334 +#include <loc/country-list.h>
335 +#include <loc/private.h>
337 +struct loc_country_list {
338 + struct loc_ctx* ctx;
341 + struct loc_country** elements;
342 + size_t elements_size;
347 +static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
348 + DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
349 + list, size, list->elements_size + size);
351 + struct loc_country** elements = reallocarray(list->elements,
352 + list->elements_size + size, sizeof(*list->elements));
356 + list->elements = elements;
357 + list->elements_size += size;
362 +LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
363 + struct loc_country_list** list) {
364 + struct loc_country_list* l = calloc(1, sizeof(*l));
368 + l->ctx = loc_ref(ctx);
371 + DEBUG(l->ctx, "Country list allocated at %p\n", l);
377 +LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
383 +static void loc_country_list_free(struct loc_country_list* list) {
384 + DEBUG(list->ctx, "Releasing country list at %p\n", list);
386 + loc_country_list_clear(list);
388 + loc_unref(list->ctx);
392 +LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
396 + if (--list->refcount > 0)
399 + loc_country_list_free(list);
403 +LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
407 +LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
408 + return list->size == 0;
411 +LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
412 + if (!list->elements)
415 + for (unsigned int i = 0; i < list->size; i++)
416 + loc_country_unref(list->elements[i]);
418 + free(list->elements);
419 + list->elements = NULL;
420 + list->elements_size = 0;
425 +LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
427 + if (index >= list->size)
430 + return loc_country_ref(list->elements[index]);
433 +LOC_EXPORT int loc_country_list_append(
434 + struct loc_country_list* list, struct loc_country* country) {
435 + if (loc_country_list_contains(list, country))
438 + // Check if we have space left
439 + if (list->size >= list->elements_size) {
440 + int r = loc_country_list_grow(list, 64);
445 + DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
447 + list->elements[list->size++] = loc_country_ref(country);
452 +LOC_EXPORT int loc_country_list_contains(
453 + struct loc_country_list* list, struct loc_country* country) {
454 + for (unsigned int i = 0; i < list->size; i++) {
455 + if (loc_country_cmp(country, list->elements[i]) == 0)
462 +LOC_EXPORT int loc_country_list_contains_code(
463 + struct loc_country_list* list, const char* code) {
464 + struct loc_country* country;
466 + int r = loc_country_new(list->ctx, &country, code);
470 + r = loc_country_list_contains(list, country);
471 + loc_country_unref(country);
475 diff --git a/src/country.c b/src/country.c
476 index 2ba93e6..7aac0db 100644
479 @@ -34,6 +34,9 @@ struct loc_country {
482 LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
483 + if (!loc_country_code_is_valid(country_code))
486 struct loc_country* c = calloc(1, sizeof(*c));
489 diff --git a/src/database.c b/src/database.c
490 index fa1dad0..4b8bf1d 100644
495 #include <loc/libloc.h>
497 +#include <loc/as-list.h>
498 #include <loc/compat.h>
499 #include <loc/country.h>
500 +#include <loc/country-list.h>
501 #include <loc/database.h>
502 #include <loc/format.h>
503 #include <loc/network.h>
504 @@ -99,11 +101,14 @@ struct loc_database_enumerator {
508 - char country_code[3];
510 + struct loc_country_list* countries;
511 + struct loc_as_list* asns;
512 enum loc_network_flags flags;
518 // Index of the AS we are looking at
519 unsigned int as_index;
521 @@ -115,6 +120,9 @@ struct loc_database_enumerator {
522 struct loc_node_stack network_stack[MAX_STACK_DEPTH];
523 int network_stack_depth;
524 unsigned int* networks_visited;
526 + // For subnet search
527 + struct loc_network_list* stack;
530 static int loc_database_read_magic(struct loc_database* db) {
531 @@ -242,11 +250,11 @@ static int loc_database_read_signature(struct loc_database* db,
532 char** dst, char* src, size_t length) {
533 // Check for a plausible signature length
534 if (length > LOC_SIGNATURE_MAX_LENGTH) {
535 - ERROR(db->ctx, "Signature too long: %ld\n", length);
536 + ERROR(db->ctx, "Signature too long: %zu\n", length);
540 - DEBUG(db->ctx, "Reading signature of %ld bytes\n", length);
541 + DEBUG(db->ctx, "Reading signature of %zu bytes\n", length);
544 *dst = malloc(length);
545 @@ -611,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
548 clock_t end = clock();
549 - DEBUG(db->ctx, "Signature checked in %.4fms\n",
550 + INFO(db->ctx, "Signature checked in %.4fms\n",
551 (double)(end - start) / CLOCKS_PER_SEC * 1000);
554 @@ -671,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
556 off_t hi = db->as_count - 1;
560 clock_t start = clock();
564 off_t i = (lo + hi) / 2;
565 @@ -685,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
566 // Check if this is a match
567 uint32_t as_number = loc_as_get_number(*as);
568 if (as_number == number) {
570 clock_t end = clock();
572 // Log how fast this has been
573 DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
574 (double)(end - start) / CLOCKS_PER_SEC * 1000);
579 @@ -733,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
585 char* string = loc_network_str(*network);
586 DEBUG(db->ctx, "Got network %s\n", string);
593 @@ -762,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru
596 // Check if the given IP address is inside the network
597 - r = loc_network_match_address(*network, address);
599 + if (!loc_network_match_address(*network, address)) {
600 DEBUG(db->ctx, "Searched address is not part of the network\n");
602 loc_network_unref(*network);
603 @@ -832,17 +845,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
609 clock_t start = clock();
612 int r = __loc_database_lookup(db, address, network, &network_address,
613 db->network_nodes_v1, 0);
616 clock_t end = clock();
618 // Log how fast this has been
619 DEBUG(db->ctx, "Executed network search in %.4fms\n",
620 (double)(end - start) / CLOCKS_PER_SEC * 1000);
625 @@ -889,8 +906,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
627 off_t hi = db->countries_count - 1;
631 clock_t start = clock();
635 off_t i = (lo + hi) / 2;
636 @@ -905,11 +924,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
637 int result = strcmp(code, cc);
641 clock_t end = clock();
643 // Log how fast this has been
644 DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
645 (double)(end - start) / CLOCKS_PER_SEC * 1000);
650 @@ -932,8 +953,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
654 +static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
655 + DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
657 + // Release all references
658 + loc_database_unref(enumerator->db);
659 + loc_unref(enumerator->ctx);
661 + if (enumerator->string)
662 + free(enumerator->string);
664 + if (enumerator->countries)
665 + loc_country_list_unref(enumerator->countries);
667 + if (enumerator->asns)
668 + loc_as_list_unref(enumerator->asns);
670 + // Free network search
671 + free(enumerator->networks_visited);
673 + // Free subnet stack
674 + if (enumerator->stack)
675 + loc_network_list_unref(enumerator->stack);
680 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
681 - struct loc_database* db, enum loc_database_enumerator_mode mode) {
682 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
683 struct loc_database_enumerator* e = calloc(1, sizeof(*e));
686 @@ -944,11 +991,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
691 + e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
693 // Initialise graph search
694 - //e->network_stack[++e->network_stack_depth] = 0;
695 e->network_stack_depth = 1;
696 e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
699 + int r = loc_network_list_new(e->ctx, &e->stack);
701 + loc_database_enumerator_free(e);
705 DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
708 @@ -961,22 +1017,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
712 -static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
713 - DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
715 - // Release all references
716 - loc_database_unref(enumerator->db);
717 - loc_unref(enumerator->ctx);
719 - if (enumerator->string)
720 - free(enumerator->string);
722 - // Free network search
723 - free(enumerator->networks_visited);
728 LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
731 @@ -998,40 +1038,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
735 -LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
736 - // Set empty country code
737 - if (!country_code || !*country_code) {
738 - *enumerator->country_code = '\0';
741 +LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
742 + struct loc_database_enumerator* enumerator) {
743 + if (!enumerator->countries)
746 - // Treat A1, A2, A3 as special country codes,
747 - // but perform search for flags instead
748 - if (strcmp(country_code, "A1") == 0) {
749 - return loc_database_enumerator_set_flag(enumerator,
750 - LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
751 - } else if (strcmp(country_code, "A2") == 0) {
752 - return loc_database_enumerator_set_flag(enumerator,
753 - LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
754 - } else if (strcmp(country_code, "A3") == 0) {
755 - return loc_database_enumerator_set_flag(enumerator,
756 - LOC_NETWORK_FLAG_ANYCAST);
758 + return loc_country_list_ref(enumerator->countries);
761 - // Country codes must be two characters
762 - if (!loc_country_code_is_valid(country_code))
764 +LOC_EXPORT int loc_database_enumerator_set_countries(
765 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
766 + if (enumerator->countries)
767 + loc_country_list_unref(enumerator->countries);
769 - for (unsigned int i = 0; i < 3; i++) {
770 - enumerator->country_code[i] = country_code[i];
772 + enumerator->countries = loc_country_list_ref(countries);
777 -LOC_EXPORT int loc_database_enumerator_set_asn(
778 - struct loc_database_enumerator* enumerator, unsigned int asn) {
779 - enumerator->asn = asn;
780 +LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
781 + struct loc_database_enumerator* enumerator) {
782 + if (!enumerator->asns)
785 + return loc_as_list_ref(enumerator->asns);
788 +LOC_EXPORT int loc_database_enumerator_set_asns(
789 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
790 + if (enumerator->asns)
791 + loc_as_list_unref(enumerator->asns);
793 + enumerator->asns = loc_as_list_ref(asns);
797 @@ -1110,16 +1148,64 @@ static int loc_database_enumerator_stack_push_node(
801 -LOC_EXPORT int loc_database_enumerator_next_network(
802 - struct loc_database_enumerator* enumerator, struct loc_network** network) {
805 +static int loc_database_enumerator_filter_network(
806 + struct loc_database_enumerator* enumerator, struct loc_network* network) {
807 + // Skip if the family does not match
808 + if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
809 + DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
813 - // Do not do anything if not in network mode
814 - if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
816 + // Skip if the country code does not match
817 + if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
818 + const char* country_code = loc_network_get_country_code(network);
821 + if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
822 + DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
827 + // Skip if the ASN does not match
828 + if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
829 + uint32_t asn = loc_network_get_asn(network);
831 + if (!loc_as_list_contains_number(enumerator->asns, asn)) {
832 + DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
837 + // Skip if flags do not match
838 + if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
839 + DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
847 +static int __loc_database_enumerator_next_network(
848 + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
849 + // Return top element from the stack
851 + *network = loc_network_list_pop_first(enumerator->stack);
857 + // Throw away any networks by filter
858 + if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
859 + loc_network_unref(*network);
868 DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
869 enumerator->network_stack_depth);
870 @@ -1149,7 +1235,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
871 enumerator->db->network_nodes_v1 + node->offset;
873 // Add edges to stack
874 - r = loc_database_enumerator_stack_push_node(enumerator,
875 + int r = loc_database_enumerator_stack_push_node(enumerator,
876 be32toh(n->one), 1, node->depth + 1);
879 @@ -1175,54 +1261,142 @@ LOC_EXPORT int loc_database_enumerator_next_network(
883 - // Check if we are interested in this network
884 + // Return all networks when the filter is disabled
888 - // Skip if the family does not match
889 - if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
890 + // Check if we are interested in this network
891 + if (loc_database_enumerator_filter_network(enumerator, *network)) {
892 loc_network_unref(*network);
898 - // Skip if the country code does not match
899 - if (*enumerator->country_code &&
900 - !loc_network_match_country_code(*network, enumerator->country_code)) {
901 - loc_network_unref(*network);
909 + // Reached the end of the search
913 - // Skip if the ASN does not match
914 - if (enumerator->asn &&
915 - !loc_network_match_asn(*network, enumerator->asn)) {
916 - loc_network_unref(*network);
918 +static int __loc_database_enumerator_next_network_flattened(
919 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
920 + // Fetch the next network
921 + int r = __loc_database_enumerator_next_network(enumerator, network, 1);
927 + // End if we could not read another network
931 - // Skip if flags do not match
932 - if (enumerator->flags &&
933 - !loc_network_match_flag(*network, enumerator->flags)) {
934 - loc_network_unref(*network);
936 + struct loc_network* subnet = NULL;
937 + struct loc_network_list* subnets;
940 + // Create a list with all subnets
941 + r = loc_network_list_new(enumerator->ctx, &subnets);
945 + // Search all subnets from the database
947 + // Fetch the next network in line
948 + r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
950 + loc_network_unref(subnet);
951 + loc_network_list_unref(subnets);
956 + // End if we did not receive another subnet
960 + // Collect all subnets in a list
961 + if (loc_network_is_subnet(*network, subnet)) {
962 + r = loc_network_list_push(subnets, subnet);
964 + loc_network_unref(subnet);
965 + loc_network_list_unref(subnets);
971 + loc_network_unref(subnet);
975 + // If this is not a subnet, we push it back onto the stack and break
976 + r = loc_network_list_push(enumerator->stack, subnet);
978 + loc_network_unref(subnet);
979 + loc_network_list_unref(subnets);
984 + loc_network_unref(subnet);
988 - // Reached the end of the search
989 + DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
991 - // Mark all nodes as non-visited
992 - for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
993 - enumerator->networks_visited[i] = 0;
994 + // We can abort here if the network has no subnets
995 + if (loc_network_list_empty(subnets)) {
996 + loc_network_list_unref(subnets);
1002 + // If the network has any subnets, we will break it into smaller parts
1003 + // without the subnets.
1004 + struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
1006 + loc_network_list_unref(subnets);
1010 + // Merge subnets onto the stack
1011 + r = loc_network_list_merge(enumerator->stack, subnets);
1013 + loc_network_list_unref(subnets);
1014 + loc_network_list_unref(excluded);
1019 + // Push excluded list onto the stack
1020 + r = loc_network_list_merge(enumerator->stack, excluded);
1022 + loc_network_list_unref(subnets);
1023 + loc_network_list_unref(excluded);
1028 + loc_network_list_unref(subnets);
1029 + loc_network_list_unref(excluded);
1031 + // Drop the network and restart the whole process again to pick the next network
1032 + loc_network_unref(*network);
1034 + return __loc_database_enumerator_next_network_flattened(enumerator, network);
1037 +LOC_EXPORT int loc_database_enumerator_next_network(
1038 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
1039 + // Do not do anything if not in network mode
1040 + if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
1043 + // Flatten output?
1044 + if (enumerator->flatten)
1045 + return __loc_database_enumerator_next_network_flattened(enumerator, network);
1047 + return __loc_database_enumerator_next_network(enumerator, network, 1);
1050 LOC_EXPORT int loc_database_enumerator_next_country(
1051 diff --git a/src/libloc.sym b/src/libloc.sym
1052 index b8296eb..ee333f1 100644
1053 --- a/src/libloc.sym
1054 +++ b/src/libloc.sym
1055 @@ -37,6 +37,18 @@ global:
1060 + loc_as_list_append;
1061 + loc_as_list_clear;
1062 + loc_as_list_contains;
1063 + loc_as_list_contains_number;
1064 + loc_as_list_empty;
1069 + loc_as_list_unref;
1073 loc_country_code_is_valid;
1074 @@ -49,6 +61,18 @@ global:
1075 loc_country_set_name;
1079 + loc_country_list_append;
1080 + loc_country_list_clear;
1081 + loc_country_list_contains;
1082 + loc_country_list_contains_code;
1083 + loc_country_list_empty;
1084 + loc_country_list_get;
1085 + loc_country_list_new;
1086 + loc_country_list_ref;
1087 + loc_country_list_size;
1088 + loc_country_list_unref;
1091 loc_database_add_as;
1092 loc_database_count_as;
1093 @@ -66,13 +90,15 @@ global:
1094 loc_database_verify;
1096 # Database Enumerator
1097 + loc_database_enumerator_get_asns;
1098 + loc_database_enumerator_get_countries;
1099 loc_database_enumerator_new;
1100 loc_database_enumerator_next_as;
1101 loc_database_enumerator_next_country;
1102 loc_database_enumerator_next_network;
1103 loc_database_enumerator_ref;
1104 - loc_database_enumerator_set_asn;
1105 - loc_database_enumerator_set_country_code;
1106 + loc_database_enumerator_set_asns;
1107 + loc_database_enumerator_set_countries;
1108 loc_database_enumerator_set_family;
1109 loc_database_enumerator_set_flag;
1110 loc_database_enumerator_set_string;
1111 @@ -80,24 +106,48 @@ global:
1114 loc_network_address_family;
1116 + loc_network_exclude;
1117 + loc_network_exclude_list;
1118 loc_network_format_first_address;
1119 loc_network_format_last_address;
1120 loc_network_get_asn;
1121 loc_network_get_country_code;
1122 + loc_network_get_first_address;
1123 + loc_network_get_last_address;
1124 loc_network_has_flag;
1125 - loc_network_is_subnet_of;
1126 + loc_network_is_subnet;
1127 + loc_network_match_address;
1128 loc_network_match_asn;
1129 loc_network_match_country_code;
1130 loc_network_match_flag;
1132 loc_network_new_from_string;
1133 + loc_network_overlaps;
1134 + loc_network_prefix;
1136 loc_network_set_asn;
1137 loc_network_set_country_code;
1138 loc_network_set_flag;
1140 + loc_network_subnets;
1144 + loc_network_list_clear;
1145 + loc_network_list_contains;
1146 + loc_network_list_dump;
1147 + loc_network_list_empty;
1148 + loc_network_list_get;
1149 + loc_network_list_merge;
1150 + loc_network_list_new;
1151 + loc_network_list_pop;
1152 + loc_network_list_pop_first;
1153 + loc_network_list_push;
1154 + loc_network_list_ref;
1155 + loc_network_list_size;
1156 + loc_network_list_unref;
1160 loc_writer_add_country;
1161 diff --git a/src/loc/as-list.h b/src/loc/as-list.h
1162 new file mode 100644
1163 index 0000000..7b5c4e8
1165 +++ b/src/loc/as-list.h
1168 + libloc - A library to determine the location of someone on the Internet
1170 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
1172 + This library is free software; you can redistribute it and/or
1173 + modify it under the terms of the GNU Lesser General Public
1174 + License as published by the Free Software Foundation; either
1175 + version 2.1 of the License, or (at your option) any later version.
1177 + This library is distributed in the hope that it will be useful,
1178 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1179 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1180 + Lesser General Public License for more details.
1183 +#ifndef LIBLOC_AS_LIST_H
1184 +#define LIBLOC_AS_LIST_H
1186 +#include <loc/as.h>
1187 +#include <loc/libloc.h>
1189 +struct loc_as_list;
1191 +int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
1192 +struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
1193 +struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
1195 +size_t loc_as_list_size(struct loc_as_list* list);
1196 +int loc_as_list_empty(struct loc_as_list* list);
1197 +void loc_as_list_clear(struct loc_as_list* list);
1199 +struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
1200 +int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
1202 +int loc_as_list_contains(
1203 + struct loc_as_list* list, struct loc_as* as);
1204 +int loc_as_list_contains_number(
1205 + struct loc_as_list* list, uint32_t number);
1208 diff --git a/src/loc/country-list.h b/src/loc/country-list.h
1209 new file mode 100644
1210 index 0000000..a7f818a
1212 +++ b/src/loc/country-list.h
1215 + libloc - A library to determine the location of someone on the Internet
1217 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
1219 + This library is free software; you can redistribute it and/or
1220 + modify it under the terms of the GNU Lesser General Public
1221 + License as published by the Free Software Foundation; either
1222 + version 2.1 of the License, or (at your option) any later version.
1224 + This library is distributed in the hope that it will be useful,
1225 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1226 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1227 + Lesser General Public License for more details.
1230 +#ifndef LIBLOC_COUNTRY_LIST_H
1231 +#define LIBLOC_COUNTRY_LIST_H
1233 +#include <stdlib.h>
1235 +#include <loc/libloc.h>
1236 +#include <loc/country.h>
1238 +struct loc_country_list;
1240 +int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
1241 +struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
1242 +struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
1244 +size_t loc_country_list_size(struct loc_country_list* list);
1245 +int loc_country_list_empty(struct loc_country_list* list);
1246 +void loc_country_list_clear(struct loc_country_list* list);
1248 +struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
1249 +int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
1251 +int loc_country_list_contains(
1252 + struct loc_country_list* list, struct loc_country* country);
1253 +int loc_country_list_contains_code(
1254 + struct loc_country_list* list, const char* code);
1257 diff --git a/src/loc/database.h b/src/loc/database.h
1258 index 43173dd..70801f0 100644
1259 --- a/src/loc/database.h
1260 +++ b/src/loc/database.h
1262 #include <loc/network.h>
1264 #include <loc/country.h>
1265 +#include <loc/country-list.h>
1267 struct loc_database;
1268 int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
1269 @@ -55,15 +56,24 @@ enum loc_database_enumerator_mode {
1270 LOC_DB_ENUMERATE_COUNTRIES = 3,
1273 +enum loc_database_enumerator_flags {
1274 + LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
1277 struct loc_database_enumerator;
1278 int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
1279 - struct loc_database* db, enum loc_database_enumerator_mode mode);
1280 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
1281 struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
1282 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
1284 int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
1285 -int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
1286 -int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
1287 +struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
1288 +int loc_database_enumerator_set_countries(
1289 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
1290 +struct loc_as_list* loc_database_enumerator_get_asns(
1291 + struct loc_database_enumerator* enumerator);
1292 +int loc_database_enumerator_set_asns(
1293 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
1294 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
1295 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
1296 int loc_database_enumerator_next_as(
1297 diff --git a/src/loc/network-list.h b/src/loc/network-list.h
1298 new file mode 100644
1299 index 0000000..bee21c4
1301 +++ b/src/loc/network-list.h
1304 + libloc - A library to determine the location of someone on the Internet
1306 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
1308 + This library is free software; you can redistribute it and/or
1309 + modify it under the terms of the GNU Lesser General Public
1310 + License as published by the Free Software Foundation; either
1311 + version 2.1 of the License, or (at your option) any later version.
1313 + This library is distributed in the hope that it will be useful,
1314 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1315 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1316 + Lesser General Public License for more details.
1319 +#ifndef LIBLOC_NETWORK_LIST_H
1320 +#define LIBLOC_NETWORK_LIST_H
1322 +#include <loc/network.h>
1324 +struct loc_network_list;
1325 +int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
1326 +struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
1327 +struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
1328 +size_t loc_network_list_size(struct loc_network_list* list);
1329 +int loc_network_list_empty(struct loc_network_list* list);
1330 +void loc_network_list_clear(struct loc_network_list* list);
1331 +void loc_network_list_dump(struct loc_network_list* list);
1332 +struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
1333 +int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
1334 +struct loc_network* loc_network_list_pop(struct loc_network_list* list);
1335 +struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
1336 +int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
1337 +int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
1340 diff --git a/src/loc/network.h b/src/loc/network.h
1341 index 70c3803..af3dafd 100644
1342 --- a/src/loc/network.h
1343 +++ b/src/loc/network.h
1346 #include <loc/libloc.h>
1347 #include <loc/format.h>
1348 +#include <loc/network-list.h>
1350 enum loc_network_flags {
1351 LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
1352 @@ -37,8 +38,11 @@ struct loc_network* loc_network_ref(struct loc_network* network);
1353 struct loc_network* loc_network_unref(struct loc_network* network);
1354 char* loc_network_str(struct loc_network* network);
1355 int loc_network_address_family(struct loc_network* network);
1356 +unsigned int loc_network_prefix(struct loc_network* network);
1358 +const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
1359 char* loc_network_format_first_address(struct loc_network* network);
1360 +const struct in6_addr* loc_network_get_last_address(struct loc_network* network);
1361 char* loc_network_format_last_address(struct loc_network* network);
1362 int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
1364 @@ -54,7 +58,14 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
1365 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
1366 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
1368 -int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
1369 +int loc_network_cmp(struct loc_network* self, struct loc_network* other);
1370 +int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
1371 +int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
1372 +int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
1373 +struct loc_network_list* loc_network_exclude(
1374 + struct loc_network* self, struct loc_network* other);
1375 +struct loc_network_list* loc_network_exclude_list(
1376 + struct loc_network* network, struct loc_network_list* list);
1378 #ifdef LIBLOC_PRIVATE
1380 diff --git a/src/network-list.c b/src/network-list.c
1381 new file mode 100644
1382 index 0000000..698d3ab
1384 +++ b/src/network-list.c
1387 + libloc - A library to determine the location of someone on the Internet
1389 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
1391 + This library is free software; you can redistribute it and/or
1392 + modify it under the terms of the GNU Lesser General Public
1393 + License as published by the Free Software Foundation; either
1394 + version 2.1 of the License, or (at your option) any later version.
1396 + This library is distributed in the hope that it will be useful,
1397 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1398 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1399 + Lesser General Public License for more details.
1403 +#include <stdlib.h>
1406 +#include <loc/libloc.h>
1407 +#include <loc/network.h>
1408 +#include <loc/private.h>
1410 +struct loc_network_list {
1411 + struct loc_ctx* ctx;
1414 + struct loc_network** elements;
1415 + size_t elements_size;
1420 +static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
1421 + DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
1422 + list, size, list->elements_size + size);
1424 + struct loc_network** elements = reallocarray(list->elements,
1425 + list->elements_size + size, sizeof(*list->elements));
1429 + list->elements = elements;
1430 + list->elements_size += size;
1435 +LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
1436 + struct loc_network_list** list) {
1437 + struct loc_network_list* l = calloc(1, sizeof(*l));
1441 + l->ctx = loc_ref(ctx);
1444 + DEBUG(l->ctx, "Network list allocated at %p\n", l);
1449 +LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
1455 +static void loc_network_list_free(struct loc_network_list* list) {
1456 + DEBUG(list->ctx, "Releasing network list at %p\n", list);
1458 + // Remove all content
1459 + loc_network_list_clear(list);
1461 + loc_unref(list->ctx);
1465 +LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
1469 + if (--list->refcount > 0)
1472 + loc_network_list_free(list);
1476 +LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
1477 + return list->size;
1480 +LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
1481 + return list->size == 0;
1484 +LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
1485 + if (!list->elements)
1488 + for (unsigned int i = 0; i < list->size; i++)
1489 + loc_network_unref(list->elements[i]);
1491 + free(list->elements);
1492 + list->elements = NULL;
1493 + list->elements_size = 0;
1498 +LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
1499 + struct loc_network* network;
1502 + for (unsigned int i = 0; i < list->size; i++) {
1503 + network = list->elements[i];
1505 + s = loc_network_str(network);
1507 + INFO(list->ctx, "%4d: %s\n", i, s);
1512 +LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
1514 + if (index >= list->size)
1517 + return loc_network_ref(list->elements[index]);
1520 +static off_t loc_network_list_find(struct loc_network_list* list,
1521 + struct loc_network* network, int* found) {
1522 + // Insert at the beginning for an empty list
1523 + if (loc_network_list_empty(list))
1527 + off_t hi = list->size - 1;
1530 + // Since we are working on an ordered list, there is often a good chance that
1531 + // the network we are looking for is at the end or has to go to the end.
1533 + result = loc_network_cmp(network, list->elements[hi]);
1535 + // Match, so we are done
1536 + if (result == 0) {
1541 + // This needs to be added after the last one
1542 + } else if (result > 0) {
1549 +#ifdef ENABLE_DEBUG
1550 + // Save start time
1551 + clock_t start = clock();
1556 + while (lo <= hi) {
1557 + i = (lo + hi) / 2;
1559 + // Check if this is a match
1560 + result = loc_network_cmp(network, list->elements[i]);
1562 + if (result == 0) {
1565 +#ifdef ENABLE_DEBUG
1566 + clock_t end = clock();
1568 + // Log how fast this has been
1569 + DEBUG(list->ctx, "Found network in %.4fms at %jd\n",
1570 + (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
1586 +#ifdef ENABLE_DEBUG
1587 + clock_t end = clock();
1589 + // Log how fast this has been
1590 + DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n",
1591 + (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
1597 +LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
1600 + off_t index = loc_network_list_find(list, network, &found);
1602 + // The network has been found on the list. Nothing to do.
1606 + DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n",
1607 + list, network, (intmax_t)index);
1609 + // Check if we have space left
1610 + if (list->size >= list->elements_size) {
1611 + int r = loc_network_list_grow(list, 64);
1616 + // The list is now larger
1619 + // Move all elements out of the way
1620 + for (unsigned int i = list->size - 1; i > index; i--)
1621 + list->elements[i] = list->elements[i - 1];
1623 + // Add the new element at the right place
1624 + list->elements[index] = loc_network_ref(network);
1629 +LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
1630 + // Return nothing when empty
1631 + if (loc_network_list_empty(list)) {
1632 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
1636 + struct loc_network* network = list->elements[--list->size];
1638 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
1643 +LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
1644 + // Return nothing when empty
1645 + if (loc_network_list_empty(list)) {
1646 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
1650 + struct loc_network* network = list->elements[0];
1652 + // Move all elements to the top of the stack
1653 + for (unsigned int i = 0; i < list->size - 1; i++) {
1654 + list->elements[i] = list->elements[i+1];
1657 + // The list is shorter now
1660 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
1665 +LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
1668 + loc_network_list_find(list, network, &found);
1673 +LOC_EXPORT int loc_network_list_merge(
1674 + struct loc_network_list* self, struct loc_network_list* other) {
1677 + for (unsigned int i = 0; i < other->size; i++) {
1678 + r = loc_network_list_push(self, other->elements[i]);
1685 diff --git a/src/network.c b/src/network.c
1686 index 366caa2..a6b679c 100644
1690 #include <loc/compat.h>
1691 #include <loc/country.h>
1692 #include <loc/network.h>
1693 +#include <loc/network-list.h>
1694 #include <loc/private.h>
1696 struct loc_network {
1697 @@ -97,6 +98,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
1701 +static struct in6_addr address_increment(const struct in6_addr* address) {
1702 + struct in6_addr a = *address;
1704 + for (int octet = 15; octet >= 0; octet--) {
1705 + if (a.s6_addr[octet] < 255) {
1706 + a.s6_addr[octet]++;
1709 + a.s6_addr[octet] = 0;
1716 LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
1717 struct in6_addr* address, unsigned int prefix) {
1718 // Address cannot be unspecified
1719 @@ -160,9 +176,11 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
1720 LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
1721 const char* address_string) {
1722 struct in6_addr first_address;
1723 - unsigned int prefix = 0;
1724 char* prefix_string;
1726 + unsigned int prefix = 128;
1729 + DEBUG(ctx, "Attempting to parse network %s\n", address_string);
1731 // Make a copy of the string to work on it
1732 char* buffer = strdup(address_string);
1733 @@ -171,29 +189,40 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1734 // Split address and prefix
1735 address_string = strsep(&prefix_string, "/");
1737 - // Did we find a prefix?
1738 + DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
1740 + // Parse the address
1741 + r = loc_parse_address(ctx, address_string, &first_address);
1743 + DEBUG(ctx, "The address could not be parsed\n");
1747 + // If a prefix was given, we will try to parse it
1748 if (prefix_string) {
1749 // Convert prefix to integer
1750 prefix = strtol(prefix_string, NULL, 10);
1753 - // Parse the address
1754 - r = loc_parse_address(ctx, address_string, &first_address);
1756 - // Map the prefix to IPv6 if needed
1757 - if (IN6_IS_ADDR_V4MAPPED(&first_address))
1760 + DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
1764 + // Map the prefix to IPv6 if needed
1765 + if (IN6_IS_ADDR_V4MAPPED(&first_address))
1770 // Free temporary buffer
1774 - r = loc_network_new(ctx, network, &first_address, prefix);
1776 + // Exit if the parsing was unsuccessful
1781 + // Create a new network
1782 + return loc_network_new(ctx, network, &first_address, prefix);
1785 LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
1786 @@ -281,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) {
1787 return network->family;
1790 +LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) {
1791 + switch (network->family) {
1793 + return network->prefix;
1796 + return network->prefix - 96;
1802 static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) {
1803 const size_t length = INET6_ADDRSTRLEN;
1805 @@ -314,10 +355,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc
1809 +LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) {
1810 + return &network->first_address;
1813 LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) {
1814 return loc_network_format_address(network, &network->first_address);
1817 +LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) {
1818 + return &network->last_address;
1821 LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
1822 return loc_network_format_address(network, &network->last_address);
1824 @@ -325,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
1825 LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) {
1826 // Address must be larger than the start address
1827 if (in6_addr_cmp(&network->first_address, address) > 0)
1831 // Address must be smaller than the last address
1832 if (in6_addr_cmp(&network->last_address, address) < 0)
1836 // The address is inside this network
1841 LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) {
1842 @@ -392,20 +441,310 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
1843 return loc_network_has_flag(network, flag);
1846 -LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
1847 +LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
1848 + // Compare address
1849 + int r = in6_addr_cmp(&self->first_address, &other->first_address);
1854 + if (self->prefix > other->prefix)
1856 + else if (self->prefix < other->prefix)
1859 + // Both networks are equal
1863 +LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
1864 + // Either of the start addresses must be in the other subnet
1865 + if (loc_network_match_address(self, &other->first_address))
1868 + if (loc_network_match_address(other, &self->first_address))
1871 + // Or either of the end addresses is in the other subnet
1872 + if (loc_network_match_address(self, &other->last_address))
1875 + if (loc_network_match_address(other, &self->last_address))
1881 +LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
1882 + // The prefix must be smaller (this avoids the more complex comparisons later)
1883 + if (self->prefix > other->prefix)
1886 // If the start address of the other network is smaller than this network,
1887 // it cannot be a subnet.
1888 - if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
1889 + if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
1892 // If the end address of the other network is greater than this network,
1893 // it cannot be a subnet.
1894 - if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
1895 + if (in6_addr_cmp(&self->last_address, &other->last_address) < 0)
1901 +LOC_EXPORT int loc_network_subnets(struct loc_network* network,
1902 + struct loc_network** subnet1, struct loc_network** subnet2) {
1907 + // New prefix length
1908 + unsigned int prefix = network->prefix + 1;
1910 + // Check if the new prefix is valid
1911 + if (valid_prefix(&network->first_address, prefix))
1914 + // Create the first half of the network
1915 + r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix);
1919 + // The next subnet starts after the first one
1920 + struct in6_addr first_address = address_increment(&(*subnet1)->last_address);
1922 + // Create the second half of the network
1923 + r = loc_network_new(network->ctx, subnet2, &first_address, prefix);
1927 + // Copy country code
1928 + const char* country_code = loc_network_get_country_code(network);
1929 + if (country_code) {
1930 + loc_network_set_country_code(*subnet1, country_code);
1931 + loc_network_set_country_code(*subnet2, country_code);
1935 + uint32_t asn = loc_network_get_asn(network);
1937 + loc_network_set_asn(*subnet1, asn);
1938 + loc_network_set_asn(*subnet2, asn);
1942 + loc_network_set_flag(*subnet1, network->flags);
1943 + loc_network_set_flag(*subnet2, network->flags);
1948 +static int __loc_network_exclude(struct loc_network* network,
1949 + struct loc_network* other, struct loc_network_list* list) {
1950 + struct loc_network* subnet1 = NULL;
1951 + struct loc_network* subnet2 = NULL;
1953 + int r = loc_network_subnets(network, &subnet1, &subnet2);
1957 + if (loc_network_cmp(other, subnet1) == 0) {
1958 + r = loc_network_list_push(list, subnet2);
1962 + } else if (loc_network_cmp(other, subnet2) == 0) {
1963 + r = loc_network_list_push(list, subnet1);
1967 + } else if (loc_network_is_subnet(subnet1, other)) {
1968 + r = loc_network_list_push(list, subnet2);
1972 + r = __loc_network_exclude(subnet1, other, list);
1976 + } else if (loc_network_is_subnet(subnet2, other)) {
1977 + r = loc_network_list_push(list, subnet1);
1981 + r = __loc_network_exclude(subnet2, other, list);
1986 + ERROR(network->ctx, "We should never get here\n");
1993 + loc_network_unref(subnet1);
1996 + loc_network_unref(subnet2);
2001 +static int __loc_network_exclude_to_list(struct loc_network* self,
2002 + struct loc_network* other, struct loc_network_list* list) {
2003 + // Other must be a subnet of self
2004 + if (!loc_network_is_subnet(self, other)) {
2005 + DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
2011 + // We cannot perform this operation if both networks equal
2012 + if (loc_network_cmp(self, other) == 0) {
2013 + DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
2019 + return __loc_network_exclude(self, other, list);
2022 +LOC_EXPORT struct loc_network_list* loc_network_exclude(
2023 + struct loc_network* self, struct loc_network* other) {
2024 + struct loc_network_list* list;
2026 +#ifdef ENABLE_DEBUG
2027 + char* n1 = loc_network_str(self);
2028 + char* n2 = loc_network_str(other);
2030 + DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
2036 + // Create a new list with the result
2037 + int r = loc_network_list_new(self->ctx, &list);
2039 + ERROR(self->ctx, "Could not create network list: %d\n", r);
2044 + r = __loc_network_exclude_to_list(self, other, list);
2046 + loc_network_list_unref(list);
2051 + // Return the result
2055 +LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
2056 + struct loc_network* network, struct loc_network_list* list) {
2057 + struct loc_network_list* to_check;
2059 + // Create a new list with all networks to look at
2060 + int r = loc_network_list_new(network->ctx, &to_check);
2064 + struct loc_network* subnet = NULL;
2065 + struct loc_network_list* subnets = NULL;
2067 + for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
2068 + subnet = loc_network_list_get(list, i);
2070 + // Find all excluded networks
2071 + if (!loc_network_list_contains(to_check, subnet)) {
2072 + r = __loc_network_exclude_to_list(network, subnet, to_check);
2074 + loc_network_list_unref(to_check);
2075 + loc_network_unref(subnet);
2082 + loc_network_unref(subnet);
2085 + r = loc_network_list_new(network->ctx, &subnets);
2087 + loc_network_list_unref(to_check);
2091 + off_t smallest_subnet = 0;
2093 + while (!loc_network_list_empty(to_check)) {
2094 + struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check);
2096 + // Check whether the subnet to check is part of the input list
2097 + if (loc_network_list_contains(list, subnet_to_check)) {
2098 + loc_network_unref(subnet_to_check);
2102 + // Marks whether this subnet passed all checks
2105 + for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) {
2106 + subnet = loc_network_list_get(list, i);
2108 + // Drop this subnet if is a subnet of another subnet
2109 + if (loc_network_is_subnet(subnet, subnet_to_check)) {
2111 + loc_network_unref(subnet);
2115 + // Break it down if it overlaps
2116 + if (loc_network_overlaps(subnet, subnet_to_check)) {
2119 + __loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
2121 + loc_network_unref(subnet);
2125 + // If the subnet is strictly greater, we do not need to continue the search
2126 + r = loc_network_cmp(subnet, subnet_to_check);
2128 + loc_network_unref(subnet);
2131 + // If it is strictly smaller, we can continue the search from here next
2132 + // time because all networks that are to be checked can only be larger
2134 + } else if (r < 0) {
2135 + smallest_subnet = i;
2138 + loc_network_unref(subnet);
2142 + r = loc_network_list_push(subnets, subnet_to_check);
2145 + loc_network_unref(subnet_to_check);
2148 + loc_network_list_unref(to_check);
2153 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
2155 loc_country_code_copy(dbobj->country_code, network->country_code);
2156 @@ -474,7 +813,7 @@ struct loc_network_tree_node {
2157 struct loc_network* network;
2160 -LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
2161 +int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
2162 struct loc_network_tree* t = calloc(1, sizeof(*t));
2165 @@ -494,7 +833,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
2169 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
2170 +struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
2171 return loc_network_tree_node_ref(tree->root);
2174 @@ -566,7 +905,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
2178 -LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
2179 +int loc_network_tree_walk(struct loc_network_tree* tree,
2180 int(*filter_callback)(struct loc_network* network, void* data),
2181 int(*callback)(struct loc_network* network, void* data), void* data) {
2182 return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
2183 @@ -581,7 +920,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
2187 -LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
2188 +struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
2189 if (--tree->refcount > 0)
2192 @@ -602,13 +941,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
2196 -LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
2197 +int loc_network_tree_dump(struct loc_network_tree* tree) {
2198 DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
2200 return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
2203 -LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
2204 +int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
2205 DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
2207 struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
2208 @@ -639,7 +978,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
2212 -LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
2213 +size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
2216 int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
2217 @@ -661,11 +1000,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
2221 -LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
2222 +size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
2223 return __loc_network_tree_count_nodes(tree->root);
2226 -LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
2227 +int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
2228 struct loc_network_tree_node* n = calloc(1, sizeof(*n));
2231 @@ -680,7 +1019,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
2235 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
2236 +struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
2240 @@ -703,7 +1042,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
2244 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
2245 +struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
2249 @@ -714,7 +1053,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
2253 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
2254 +struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
2258 @@ -726,10 +1065,10 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
2259 return loc_network_tree_node_ref(node);
2262 -LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
2263 +int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
2264 return (!!node->network);
2267 -LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2268 +struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2269 return loc_network_ref(node->network);
2271 diff --git a/src/perl/Location.xs b/src/perl/Location.xs
2272 index dcf3f0d..b7676d2 100644
2273 --- a/src/perl/Location.xs
2274 +++ b/src/perl/Location.xs
2275 @@ -125,7 +125,7 @@ database_countries(db)
2277 // Create Database enumerator
2278 struct loc_database_enumerator* enumerator;
2279 - int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
2280 + int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
2283 croak("Could not create a database enumerator\n");
2284 diff --git a/src/python/database.c b/src/python/database.c
2285 index 1013a58..0aa03cc 100644
2286 --- a/src/python/database.c
2287 +++ b/src/python/database.c
2291 #include <loc/libloc.h>
2292 +#include <loc/as.h>
2293 +#include <loc/as-list.h>
2294 #include <loc/database.h>
2296 #include "locationmodule.h"
2297 @@ -207,10 +209,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
2298 return (PyObject*)self;
2301 -static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
2302 +static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
2303 struct loc_database_enumerator* enumerator;
2305 - int r = loc_database_enumerator_new(&enumerator, self->db, what);
2306 + int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
2308 PyErr_SetFromErrno(PyExc_SystemError);
2310 @@ -223,7 +225,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
2313 static PyObject* Database_ases(DatabaseObject* self) {
2314 - return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
2315 + return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
2318 static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2319 @@ -234,7 +236,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2321 struct loc_database_enumerator* enumerator;
2323 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
2324 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
2326 PyErr_SetFromErrno(PyExc_SystemError);
2328 @@ -250,44 +252,142 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2331 static PyObject* Database_networks(DatabaseObject* self) {
2332 - return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
2333 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
2336 +static PyObject* Database_networks_flattened(DatabaseObject *self) {
2337 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
2340 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
2341 - char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
2342 - const char* country_code = NULL;
2343 - unsigned int asn = 0;
2344 + char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
2345 + PyObject* country_codes = NULL;
2346 + PyObject* asn_list = NULL;
2351 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
2352 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
2353 + &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
2356 struct loc_database_enumerator* enumerator;
2357 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
2358 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
2359 + (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
2361 PyErr_SetFromErrno(PyExc_SystemError);
2365 // Set country code we are searching for
2366 - if (country_code) {
2367 - r = loc_database_enumerator_set_country_code(enumerator, country_code);
2368 + if (country_codes) {
2369 + struct loc_country_list* countries;
2370 + r = loc_country_list_new(loc_ctx, &countries);
2372 + PyErr_SetString(PyExc_SystemError, "Could not create country list");
2376 + for (int i = 0; i < PyList_Size(country_codes); i++) {
2377 + PyObject* item = PyList_GetItem(country_codes, i);
2379 + if (!PyUnicode_Check(item)) {
2380 + PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
2381 + loc_country_list_unref(countries);
2385 + const char* country_code = PyUnicode_AsUTF8(item);
2387 + struct loc_country* country;
2388 + r = loc_country_new(loc_ctx, &country, country_code);
2390 + if (r == -EINVAL) {
2391 + PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
2393 + PyErr_SetString(PyExc_SystemError, "Could not create country");
2396 + loc_country_list_unref(countries);
2400 + // Append it to the list
2401 + r = loc_country_list_append(countries, country);
2403 + PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
2405 + loc_country_list_unref(countries);
2406 + loc_country_unref(country);
2410 + loc_country_unref(country);
2413 + r = loc_database_enumerator_set_countries(enumerator, countries);
2415 PyErr_SetFromErrno(PyExc_SystemError);
2417 + loc_country_list_unref(countries);
2421 + loc_country_list_unref(countries);
2424 // Set the ASN we are searching for
2426 - r = loc_database_enumerator_set_asn(enumerator, asn);
2428 + struct loc_as_list* asns;
2429 + r = loc_as_list_new(loc_ctx, &asns);
2431 + PyErr_SetString(PyExc_SystemError, "Could not create AS list");
2435 + for (int i = 0; i < PyList_Size(asn_list); i++) {
2436 + PyObject* item = PyList_GetItem(asn_list, i);
2438 + if (!PyLong_Check(item)) {
2439 + PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
2441 + loc_as_list_unref(asns);
2445 + unsigned long number = PyLong_AsLong(item);
2447 + struct loc_as* as;
2448 + r = loc_as_new(loc_ctx, &as, number);
2450 + PyErr_SetString(PyExc_SystemError, "Could not create AS");
2452 + loc_as_list_unref(asns);
2457 + r = loc_as_list_append(asns, as);
2459 + PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
2461 + loc_as_list_unref(asns);
2469 + r = loc_database_enumerator_set_asns(enumerator, asns);
2471 PyErr_SetFromErrno(PyExc_SystemError);
2473 + loc_as_list_unref(asns);
2477 + loc_as_list_unref(asns);
2480 // Set the flags we are searching for
2481 @@ -317,7 +417,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
2484 static PyObject* Database_countries(DatabaseObject* self) {
2485 - return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
2486 + return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
2489 static struct PyMethodDef Database_methods[] = {
2490 @@ -403,6 +503,13 @@ static struct PyGetSetDef Database_getsetters[] = {
2495 + "networks_flattened",
2496 + (getter)Database_networks_flattened,
2503 (getter)Database_get_vendor,
2504 diff --git a/src/python/downloader.py b/src/python/downloader.py
2505 index 87bbb68..05f7872 100644
2506 --- a/src/python/downloader.py
2507 +++ b/src/python/downloader.py
2508 @@ -119,8 +119,8 @@ class Downloader(object):
2512 - headers["If-Modified-Since"] = timestamp.strftime(
2513 - "%a, %d %b %Y %H:%M:%S GMT",
2514 + headers["If-Modified-Since"] = time.strftime(
2515 + "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
2518 t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
2519 @@ -195,7 +195,7 @@ class Downloader(object):
2520 db = Database(f.name)
2522 # Database is not recent
2523 - if timestamp and db.created_at < timestamp.timestamp():
2524 + if timestamp and db.created_at < timestamp:
2527 log.info("Downloaded new database from %s" % (time.strftime(
2528 diff --git a/src/python/export.py b/src/python/export.py
2529 index d15c6f0..f0eae26 100644
2530 --- a/src/python/export.py
2531 +++ b/src/python/export.py
2532 @@ -29,7 +29,7 @@ import _location
2533 log = logging.getLogger("location.export")
2538 _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
2539 _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
2540 _location.NETWORK_FLAG_ANYCAST : "A3",
2541 @@ -39,11 +39,8 @@ class OutputWriter(object):
2545 - def __init__(self, f, prefix=None, flatten=True):
2546 - self.f, self.prefix, self.flatten = f, prefix, flatten
2548 - # The previously written network
2549 - self._last_network = None
2550 + def __init__(self, f, prefix=None):
2551 + self.f, self.prefix = f, prefix
2553 # Immediately write the header
2554 self._write_header()
2555 @@ -60,18 +57,6 @@ class OutputWriter(object):
2557 return "<%s f=%s>" % (self.__class__.__name__, self.f)
2559 - def _flatten(self, network):
2561 - Checks if the given network needs to be written to file,
2562 - or if it is a subnet of the previously written network.
2564 - if self._last_network and network.is_subnet_of(self._last_network):
2567 - # Remember this network for the next call
2568 - self._last_network = network
2571 def _write_header(self):
2573 The header of the file
2574 @@ -84,16 +69,8 @@ class OutputWriter(object):
2578 - def _write_network(self, network):
2579 - self.f.write("%s\n" % network)
2581 def write(self, network):
2582 - if self.flatten and self._flatten(network):
2583 - log.debug("Skipping writing network %s" % network)
2586 - # Write the network to file
2587 - self._write_network(network)
2588 + self.f.write("%s\n" % network)
2592 @@ -114,7 +91,7 @@ class IpsetOutputWriter(OutputWriter):
2593 def _write_header(self):
2594 self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
2596 - def _write_network(self, network):
2597 + def write(self, network):
2598 self.f.write("add %s %s\n" % (self.prefix, network))
2601 @@ -130,7 +107,7 @@ class NftablesOutputWriter(OutputWriter):
2602 def _write_footer(self):
2605 - def _write_network(self, network):
2606 + def write(self, network):
2607 self.f.write(" %s,\n" % network)
2610 @@ -142,14 +119,9 @@ class XTGeoIPOutputWriter(OutputWriter):
2614 - def _write_network(self, network):
2615 - for address in (network.first_address, network.last_address):
2616 - # Convert this into a string of bits
2617 - bytes = socket.inet_pton(
2618 - network.family, address,
2621 - self.f.write(bytes)
2622 + def write(self, network):
2623 + self.f.write(network._first_address)
2624 + self.f.write(network._last_address)
2628 @@ -185,8 +157,14 @@ class Exporter(object):
2630 writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
2632 + # Filter countries from special country codes
2634 + country_code for country_code in countries if not country_code in FLAGS.values()
2637 # Get all networks that match the family
2638 - networks = self.db.search_networks(family=family)
2639 + networks = self.db.search_networks(family=family,
2640 + country_codes=country_codes, asns=asns, flatten=True)
2642 # Walk through all networks
2643 for network in networks:
2644 @@ -203,10 +181,10 @@ class Exporter(object):
2648 - for flag in flags:
2649 + for flag in FLAGS:
2650 if network.has_flag(flag):
2651 # Fetch the "fake" country code
2652 - country = flags[flag]
2653 + country = FLAGS[flag]
2656 writers[country].write(network)
2657 diff --git a/src/python/importer.py b/src/python/importer.py
2658 index f19db4b..5f46bc3 100644
2659 --- a/src/python/importer.py
2660 +++ b/src/python/importer.py
2661 @@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
2662 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
2664 # Latin America and Caribbean Network Information Centre
2665 - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2666 + "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2668 # Réseaux IP Européens
2669 #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
2670 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2671 index 1467923..2dec89e 100644
2672 --- a/src/python/location-importer.in
2673 +++ b/src/python/location-importer.in
2674 @@ -152,6 +152,7 @@ class CLI(object):
2675 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
2676 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
2677 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
2678 + CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
2681 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
2682 @@ -165,6 +166,7 @@ class CLI(object):
2684 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
2685 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
2686 + CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
2687 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
2690 @@ -188,6 +190,8 @@ class CLI(object):
2692 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
2693 ON network_overrides(network);
2694 + CREATE INDEX IF NOT EXISTS network_overrides_search
2695 + ON network_overrides USING GIST(network inet_ops);
2699 @@ -234,32 +238,24 @@ class CLI(object):
2701 # Select all known networks
2702 rows = self.db.query("""
2703 - -- Get a (sorted) list of all known networks
2704 - WITH known_networks AS (
2705 - SELECT network FROM announcements
2707 - SELECT network FROM networks
2711 -- Return a list of those networks enriched with all
2712 -- other information that we store in the database
2714 - DISTINCT ON (known_networks.network)
2715 - known_networks.network AS network,
2716 - announcements.autnum AS autnum,
2717 + DISTINCT ON (network)
2724 SELECT country FROM network_overrides overrides
2725 - WHERE announcements.network <<= overrides.network
2726 + WHERE networks.network <<= overrides.network
2727 ORDER BY masklen(overrides.network) DESC
2731 SELECT country FROM autnum_overrides overrides
2732 - WHERE announcements.autnum = overrides.number
2733 + WHERE networks.autnum = overrides.number
2737 @@ -268,50 +264,67 @@ class CLI(object):
2740 SELECT is_anonymous_proxy FROM network_overrides overrides
2741 - WHERE announcements.network <<= overrides.network
2742 + WHERE networks.network <<= overrides.network
2743 ORDER BY masklen(overrides.network) DESC
2747 SELECT is_anonymous_proxy FROM autnum_overrides overrides
2748 - WHERE announcements.autnum = overrides.number
2749 + WHERE networks.autnum = overrides.number
2752 ) AS is_anonymous_proxy,
2755 SELECT is_satellite_provider FROM network_overrides overrides
2756 - WHERE announcements.network <<= overrides.network
2757 + WHERE networks.network <<= overrides.network
2758 ORDER BY masklen(overrides.network) DESC
2762 SELECT is_satellite_provider FROM autnum_overrides overrides
2763 - WHERE announcements.autnum = overrides.number
2764 + WHERE networks.autnum = overrides.number
2767 ) AS is_satellite_provider,
2770 SELECT is_anycast FROM network_overrides overrides
2771 - WHERE announcements.network <<= overrides.network
2772 + WHERE networks.network <<= overrides.network
2773 ORDER BY masklen(overrides.network) DESC
2777 SELECT is_anycast FROM autnum_overrides overrides
2778 - WHERE announcements.autnum = overrides.number
2779 + WHERE networks.autnum = overrides.number
2784 - -- Must be part of returned values for ORDER BY clause
2785 - masklen(announcements.network) AS sort_a,
2786 - masklen(networks.network) AS sort_b
2787 - FROM known_networks
2788 - LEFT JOIN announcements ON known_networks.network <<= announcements.network
2789 - LEFT JOIN networks ON known_networks.network <<= networks.network
2790 - ORDER BY known_networks.network, sort_a DESC, sort_b DESC
2794 + known_networks.network AS network,
2795 + announcements.autnum AS autnum,
2796 + networks.country AS country,
2798 + -- Must be part of returned values for ORDER BY clause
2799 + masklen(announcements.network) AS sort_a,
2800 + masklen(networks.network) AS sort_b
2802 + SELECT network FROM announcements
2804 + SELECT network FROM networks
2806 + SELECT network FROM network_overrides
2809 + announcements ON known_networks.network <<= announcements.network
2811 + networks ON known_networks.network <<= networks.network
2813 + known_networks.network,
2820 @@ -363,6 +376,16 @@ class CLI(object):
2821 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
2823 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
2825 + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
2827 + CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
2828 + CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
2831 + # Remove all previously imported content
2832 + self.db.execute("""
2833 + TRUNCATE TABLE networks;
2836 for source in location.importer.WHOIS_SOURCES:
2837 @@ -370,31 +393,72 @@ class CLI(object):
2839 self._parse_block(block)
2841 + # Process all parsed networks from every RIR we happen to have access to,
2842 + # insert the largest network chunks into the networks table immediately...
2843 + families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
2845 + for family in (row.family for row in families):
2846 + smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
2848 + self.db.execute("INSERT INTO networks(network, country) \
2849 + SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
2851 + # ... determine any other prefixes for this network family, ...
2852 + prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
2853 + WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
2855 + # ... and insert networks with this prefix in case they provide additional
2856 + # information (i. e. subnet of a larger chunk with a different country)
2857 + for prefix in (row.prefix for row in prefixes):
2858 + self.db.execute("""
2859 + WITH candidates AS (
2866 + family(_rirdata.network) = %s
2868 + masklen(_rirdata.network) = %s
2872 + DISTINCT ON (c.network)
2875 + masklen(networks.network),
2876 + networks.country AS parent_country
2882 + c.network << networks.network
2885 + masklen(networks.network) DESC NULLS LAST
2888 + networks(network, country)
2895 + parent_country IS NULL
2897 + country <> parent_country
2898 + ON CONFLICT DO NOTHING""",
2903 INSERT INTO autnums(number, name)
2904 SELECT _autnums.number, _organizations.name FROM _autnums
2905 JOIN _organizations ON _autnums.organization = _organizations.handle
2906 - ON CONFLICT (number) DO UPDATE SET name = excluded.name
2909 - self.db.execute("""
2910 - --- Purge any redundant entries
2911 - CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
2913 - SELECT network FROM networks candidates
2915 - SELECT FROM networks
2917 - networks.network << candidates.network
2919 - networks.country = candidates.country
2922 - CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
2924 - DELETE FROM networks WHERE EXISTS (
2925 - SELECT FROM _garbage WHERE networks.network = _garbage.network
2927 + ON CONFLICT (number) DO UPDATE SET name = excluded.name;
2930 # Download all extended sources
2931 @@ -405,6 +469,69 @@ class CLI(object):
2933 self._parse_line(line)
2935 + def _check_parsed_network(self, network):
2937 + Assistive function to detect and subsequently sort out parsed
2938 + networks from RIR data (both Whois and so-called "extended sources"),
2939 + which are or have...
2941 + (a) not globally routable (RFC 1918 space, et al.)
2942 + (b) covering a too large chunk of the IP address space (prefix length
2943 + is < 7 for IPv4 networks, and < 10 for IPv6)
2944 + (c) "0.0.0.0" or "::" as a network address
2945 + (d) are too small for being publicly announced (we have decided not to
2946 + process them at the moment, as they significantly enlarge our
2947 + database without providing very helpful additional information)
2949 + This unfortunately is necessary due to brain-dead clutter across
2950 + various RIR databases, causing mismatches and eventually disruptions.
2952 + We will return False in case a network is not suitable for adding
2953 + it to our database, and True otherwise.
2956 + if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
2959 + if not network.is_global:
2960 + log.warning("Skipping non-globally routable network: %s" % network)
2963 + if network.version == 4:
2964 + if network.prefixlen < 7:
2965 + log.warning("Skipping too big IP chunk: %s" % network)
2968 + if network.prefixlen > 24:
2969 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2972 + if str(network.network_address) == "0.0.0.0":
2973 + log.warning("Skipping network based on 0.0.0.0: %s" % network)
2976 + elif network.version == 6:
2977 + if network.prefixlen < 10:
2978 + log.warning("Skipping too big IP chunk: %s" % network)
2981 + if network.prefixlen > 48:
2982 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2985 + if str(network.network_address) == "::":
2986 + log.warning("Skipping network based on '::': %s" % network)
2990 + # This should not happen...
2991 + log.warning("Skipping network of unknown family, this should not happen: %s" % network)
2994 + # In case we have made it here, the network is considered to
2995 + # be suitable for libloc consumption...
2998 def _parse_block(self, block):
2999 # Get first line to find out what type of block this is
3001 @@ -433,7 +560,7 @@ class CLI(object):
3002 autnum["asn"] = m.group(2)
3006 + autnum[key] = val.upper()
3008 # Skip empty objects
3010 @@ -447,15 +574,22 @@ class CLI(object):
3013 def _parse_inetnum_block(self, block):
3014 - logging.debug("Parsing inetnum block:")
3015 + log.debug("Parsing inetnum block:")
3019 - logging.debug(line)
3023 key, val = split_line(line)
3025 + # Filter any inetnum records which are only referring to IP space
3026 + # not managed by that specific RIR...
3027 + if key == "netname":
3028 + if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
3029 + log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
3032 if key == "inetnum":
3033 start_address, delim, end_address = val.partition("-")
3035 @@ -467,7 +601,7 @@ class CLI(object):
3036 start_address = ipaddress.ip_address(start_address)
3037 end_address = ipaddress.ip_address(end_address)
3039 - logging.warning("Could not parse line: %s" % line)
3040 + log.warning("Could not parse line: %s" % line)
3043 # Set prefix to default
3044 @@ -484,23 +618,24 @@ class CLI(object):
3047 elif key == "country":
3048 - if val == "UNITED STATES":
3051 inetnum[key] = val.upper()
3053 # Skip empty objects
3055 + if not inetnum or not "country" in inetnum:
3058 + # Skip objects with bogus country code 'ZZ'
3059 + if inetnum.get("country") == "ZZ":
3060 + log.warning("Skipping network with bogus country 'ZZ': %s" % \
3061 + (inetnum.get("inet6num") or inetnum.get("inetnum")))
3064 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
3066 - # Bail out in case we have processed a non-public IP network
3067 - if network.is_private:
3068 - logging.warning("Skipping non-globally routable network: %s" % network)
3069 + if not self._check_parsed_network(network):
3072 - self.db.execute("INSERT INTO networks(network, country) \
3073 + self.db.execute("INSERT INTO _rirdata(network, country) \
3074 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
3075 "%s" % network, inetnum.get("country"),
3077 @@ -511,7 +646,9 @@ class CLI(object):
3079 key, val = split_line(line)
3081 - if key in ("organisation", "org-name"):
3082 + if key == "organisation":
3083 + org[key] = val.upper()
3084 + elif key == "org-name":
3087 # Skip empty objects
3088 @@ -581,6 +718,9 @@ class CLI(object):
3089 log.warning("Invalid IP address: %s" % address)
3092 + if not self._check_parsed_network(network):
3095 self.db.execute("INSERT INTO networks(network, country) \
3096 VALUES(%s, %s) ON CONFLICT (network) DO \
3097 UPDATE SET country = excluded.country",
3098 diff --git a/src/python/location.in b/src/python/location.in
3099 index 44ad726..b30beae 100644
3100 --- a/src/python/location.in
3101 +++ b/src/python/location.in
3102 @@ -253,6 +253,7 @@ class CLI(object):
3103 network = db.lookup(address)
3105 print(_("Invalid IP address: %s") % address, file=sys.stderr)
3109 "address" : address,
3110 @@ -398,10 +399,7 @@ class CLI(object):
3112 def handle_update(self, db, ns):
3114 - now = datetime.datetime.utcnow()
3116 - # Parse the database timestamp
3117 - t = datetime.datetime.utcfromtimestamp(db.created_at)
3120 if ns.cron == "daily":
3121 delta = datetime.timedelta(days=1)
3122 @@ -410,22 +408,20 @@ class CLI(object):
3123 elif ns.cron == "monthly":
3124 delta = datetime.timedelta(days=30)
3126 + delta = delta.total_seconds()
3128 # Check if the database has recently been updated
3129 - if t >= (now - delta):
3130 + if db.created_at >= (now - delta):
3132 - _("The database has been updated recently (%s)") % \
3133 - format_timedelta(now - t),
3134 + _("The database has been updated recently"),
3138 # Fetch the timestamp we need from DNS
3139 t = location.discover_latest_version()
3141 - # Parse timestamp into datetime format
3142 - timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
3144 # Check the version of the local database
3145 - if db and timestamp and db.created_at >= timestamp.timestamp():
3146 + if db and t and db.created_at >= t:
3147 log.info("Already on the latest version")
3150 @@ -437,7 +433,7 @@ class CLI(object):
3152 # Try downloading a new database
3154 - t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
3155 + t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
3157 # If no file could be downloaded, log a message
3158 except FileNotFoundError as e:
3159 @@ -453,13 +449,7 @@ class CLI(object):
3163 - def handle_verify(self, ns):
3165 - db = location.Database(ns.database)
3166 - except FileNotFoundError as e:
3167 - log.error("%s: %s" % (ns.database, e))
3170 + def handle_verify(self, db, ns):
3171 # Verify the database
3172 with open(ns.public_key, "r") as f:
3173 if not db.verify(f):
3174 diff --git a/src/python/network.c b/src/python/network.c
3175 index 5496d1e..5b1369d 100644
3176 --- a/src/python/network.c
3177 +++ b/src/python/network.c
3182 +#include <limits.h>
3184 #include <loc/libloc.h>
3185 #include <loc/network.h>
3186 +#include <loc/network-list.h>
3188 #include "locationmodule.h"
3189 #include "network.h"
3191 +static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
3192 + PyObject* list = PyList_New(0);
3196 + while (!loc_network_list_empty(networks)) {
3197 + struct loc_network* network = loc_network_list_pop(networks);
3199 + PyObject* n = new_network(&NetworkType, network);
3200 + PyList_Append(list, n);
3202 + loc_network_unref(network);
3209 PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
3210 NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
3212 @@ -114,10 +134,18 @@ static int Network_set_asn(NetworkObject* self, PyObject* value) {
3213 long int asn = PyLong_AsLong(value);
3215 // Check if the ASN is within the valid range
3216 - if (asn <= 0 || asn > UINT32_MAX) {
3218 + PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn);
3222 +#if (__WORDSIZE > 32)
3223 + // Check whether the input was longer than 32 bit
3224 + if (asn > UINT32_MAX) {
3225 PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn);
3230 int r = loc_network_set_asn(self->network, asn);
3232 @@ -154,13 +182,28 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
3236 +static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
3237 + NetworkObject* other = NULL;
3239 + if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3242 + struct loc_network_list* list = loc_network_exclude(self->network, other->network);
3244 + // Convert to Python objects
3245 + PyObject* obj = PyList_FromNetworkList(list);
3246 + loc_network_list_unref(list);
3251 static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
3252 NetworkObject* other = NULL;
3254 if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3257 - if (loc_network_is_subnet_of(self->network, other->network))
3258 + if (loc_network_is_subnet(other->network, self->network))
3262 @@ -181,6 +224,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) {
3266 +static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) {
3267 + struct in_addr address4;
3269 + // Convert IPv4 addresses to struct in_addr
3270 + if (IN6_IS_ADDR_V4MAPPED(address6)) {
3271 + address4.s_addr = address6->s6_addr32[3];
3273 + return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4));
3276 + // Return IPv6 addresses as they are
3277 + return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6));
3280 +static PyObject* Network_get__first_address(NetworkObject* self) {
3281 + const struct in6_addr* address = loc_network_get_first_address(self->network);
3283 + return PyBytes_FromAddress(address);
3286 static PyObject* Network_get_last_address(NetworkObject* self) {
3287 char* address = loc_network_format_last_address(self->network);
3289 @@ -190,7 +253,19 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
3293 +static PyObject* Network_get__last_address(NetworkObject* self) {
3294 + const struct in6_addr* address = loc_network_get_last_address(self->network);
3296 + return PyBytes_FromAddress(address);
3299 static struct PyMethodDef Network_methods[] = {
3302 + (PyCFunction)Network_exclude,
3308 (PyCFunction)Network_has_flag,
3309 @@ -241,6 +316,13 @@ static struct PyGetSetDef Network_getsetters[] = {
3315 + (getter)Network_get__first_address,
3322 (getter)Network_get_last_address,
3323 @@ -248,6 +330,13 @@ static struct PyGetSetDef Network_getsetters[] = {
3329 + (getter)Network_get__last_address,
3337 diff --git a/src/test-as.c b/src/test-as.c
3338 index 839a04c..2d61675 100644
3341 @@ -95,7 +95,7 @@ int main(int argc, char** argv) {
3344 struct loc_database_enumerator* enumerator;
3345 - err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
3346 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
3348 fprintf(stderr, "Could not create a database enumerator\n");
3350 diff --git a/src/test-database.c b/src/test-database.c
3351 index b4a75c4..da4b11c 100644
3352 --- a/src/test-database.c
3353 +++ b/src/test-database.c
3354 @@ -38,6 +38,14 @@ const char* DESCRIPTION =
3355 "Maecenas ut venenatis nunc.";
3356 const char* LICENSE = "CC";
3358 +const char* networks[] = {
3360 + "2001:db8:1000::/48",
3361 + "2001:db8:2000::/48",
3362 + "2001:db8:2020::/48",
3366 static int attempt_to_open(struct loc_ctx* ctx, char* path) {
3367 FILE* f = fopen(path, "r");
3369 @@ -139,6 +147,24 @@ int main(int argc, char** argv) {
3373 + struct loc_network* network = NULL;
3375 + // Add some networks
3376 + const char** n = networks;
3378 + err = loc_writer_add_network(writer, &network, *n);
3380 + fprintf(stderr, "Could not add network %s\n", *n);
3381 + exit(EXIT_FAILURE);
3385 + loc_network_set_country_code(network, "XX");
3391 FILE* f = tmpfile();
3393 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
3394 @@ -170,6 +196,33 @@ int main(int argc, char** argv) {
3399 + struct loc_database_enumerator* enumerator;
3400 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
3402 + fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
3403 + exit(EXIT_FAILURE);
3406 + // Walk through all networks
3408 + err = loc_database_enumerator_next_network(enumerator, &network);
3410 + fprintf(stderr, "Error fetching the next network: %d\n", err);
3411 + exit(EXIT_FAILURE);
3417 + char* s = loc_network_str(network);
3418 + printf("Got network: %s\n", s);
3422 + // Free the enumerator
3423 + loc_database_enumerator_unref(enumerator);
3425 // Close the database
3426 loc_database_unref(db);
3428 diff --git a/src/test-network-list.c b/src/test-network-list.c
3429 new file mode 100644
3430 index 0000000..6f32ff7
3432 +++ b/src/test-network-list.c
3435 + libloc - A library to determine the location of someone on the Internet
3437 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
3439 + This program is free software; you can redistribute it and/or modify
3440 + it under the terms of the GNU General Public License as published by
3441 + the Free Software Foundation; either version 2 of the License, or
3442 + (at your option) any later version.
3444 + This program is distributed in the hope that it will be useful,
3445 + but WITHOUT ANY WARRANTY; without even the implied warranty of
3446 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3447 + GNU General Public License for more details.
3452 +#include <stddef.h>
3453 +#include <stdlib.h>
3454 +#include <string.h>
3455 +#include <syslog.h>
3457 +#include <loc/libloc.h>
3458 +#include <loc/network.h>
3459 +#include <loc/network-list.h>
3461 +int main(int argc, char** argv) {
3464 + struct loc_ctx* ctx;
3465 + err = loc_new(&ctx);
3467 + exit(EXIT_FAILURE);
3469 + // Enable debug logging
3470 + loc_set_log_priority(ctx, LOG_DEBUG);
3472 + // Create a network
3473 + struct loc_network* network1;
3474 + err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32");
3476 + fprintf(stderr, "Could not create the network1\n");
3477 + exit(EXIT_FAILURE);
3480 + struct loc_network* subnet1;
3481 + err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48");
3483 + fprintf(stderr, "Could not create the subnet1\n");
3484 + exit(EXIT_FAILURE);
3487 + struct loc_network* subnet2;
3488 + err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48");
3490 + fprintf(stderr, "Could not create the subnet2\n");
3491 + exit(EXIT_FAILURE);
3494 + struct loc_network* subnet3;
3495 + err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48");
3497 + fprintf(stderr, "Could not create the subnet3\n");
3498 + exit(EXIT_FAILURE);
3501 + struct loc_network* subnet4;
3502 + err = loc_network_new_from_string(ctx, &subnet4, "2001:db8:d::/48");
3504 + fprintf(stderr, "Could not create the subnet4\n");
3505 + exit(EXIT_FAILURE);
3508 + struct loc_network* subnet5;
3509 + err = loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48");
3511 + fprintf(stderr, "Could not create the subnet5\n");
3512 + exit(EXIT_FAILURE);
3515 + struct loc_network* subnet6;
3516 + err = loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48");
3518 + fprintf(stderr, "Could not create the subnet6\n");
3519 + exit(EXIT_FAILURE);
3522 + // Make a list with both subnets
3523 + struct loc_network_list* subnets;
3524 + err = loc_network_list_new(ctx, &subnets);
3526 + fprintf(stderr, "Could not create subnets list\n");
3527 + exit(EXIT_FAILURE);
3530 + size_t size = loc_network_list_size(subnets);
3532 + fprintf(stderr, "The list is not empty: %zu\n", size);
3533 + exit(EXIT_FAILURE);
3536 + err = loc_network_list_push(subnets, subnet1);
3538 + fprintf(stderr, "Could not add subnet1 to subnets list\n");
3539 + exit(EXIT_FAILURE);
3542 + if (loc_network_list_empty(subnets)) {
3543 + fprintf(stderr, "The subnets list reports that it is empty\n");
3544 + exit(EXIT_FAILURE);
3547 + err = loc_network_list_push(subnets, subnet2);
3549 + fprintf(stderr, "Could not add subnet2 to subnets list\n");
3550 + exit(EXIT_FAILURE);
3553 + // Add the fourth one next
3554 + err = loc_network_list_push(subnets, subnet4);
3556 + fprintf(stderr, "Could not add subnet4 to subnets list\n");
3557 + exit(EXIT_FAILURE);
3560 + // Add the third one
3561 + err = loc_network_list_push(subnets, subnet3);
3563 + fprintf(stderr, "Could not add subnet3 to subnets list\n");
3564 + exit(EXIT_FAILURE);
3567 + // Add more subnets
3568 + err = loc_network_list_push(subnets, subnet5);
3570 + fprintf(stderr, "Could not add subnet5 to subnets list\n");
3571 + exit(EXIT_FAILURE);
3574 + err = loc_network_list_push(subnets, subnet6);
3576 + fprintf(stderr, "Could not add subnet6 to subnets list\n");
3577 + exit(EXIT_FAILURE);
3580 + loc_network_list_dump(subnets);
3582 + size = loc_network_list_size(subnets);
3584 + fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
3585 + exit(EXIT_FAILURE);
3588 + // Exclude subnet1 from network1
3589 + struct loc_network_list* excluded = loc_network_exclude(network1, subnet1);
3591 + fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n");
3592 + exit(EXIT_FAILURE);
3595 + loc_network_list_dump(excluded);
3597 + // Exclude all subnets from network1
3598 + excluded = loc_network_exclude_list(network1, subnets);
3600 + fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n");
3601 + exit(EXIT_FAILURE);
3604 + loc_network_list_dump(excluded);
3607 + loc_network_list_unref(excluded);
3609 + loc_network_list_unref(subnets);
3610 + loc_network_unref(network1);
3611 + loc_network_unref(subnet1);
3612 + loc_network_unref(subnet2);
3615 + return EXIT_SUCCESS;
3617 diff --git a/src/test-network.c b/src/test-network.c
3618 index d38f13d..dde13f1 100644
3619 --- a/src/test-network.c
3620 +++ b/src/test-network.c
3622 GNU General Public License for more details.
3625 +#include <arpa/inet.h>
3629 @@ -37,12 +38,21 @@ int main(int argc, char** argv) {
3630 // Enable debug logging
3631 loc_set_log_priority(ctx, LOG_DEBUG);
3634 struct loc_network_tree* tree;
3635 err = loc_network_tree_new(ctx, &tree);
3637 fprintf(stderr, "Could not create the network tree\n");
3642 + struct in6_addr address;
3643 + err = inet_pton(AF_INET6, "2001:db8::1", &address);
3645 + fprintf(stderr, "Could not parse IP address\n");
3646 + exit(EXIT_FAILURE);
3650 struct loc_network* network1;
3651 @@ -58,12 +68,14 @@ int main(int argc, char** argv) {
3656 // Adding network to the tree
3657 err = loc_network_tree_add_network(tree, network1);
3659 fprintf(stderr, "Could not add network to the tree\n");
3664 // Check if the first and last addresses are correct
3665 char* string = loc_network_format_first_address(network1);
3666 @@ -88,6 +100,12 @@ int main(int argc, char** argv) {
3670 + err = loc_network_match_address(network1, &address);
3672 + fprintf(stderr, "Network1 does not match address\n");
3673 + exit(EXIT_FAILURE);
3676 struct loc_network* network2;
3677 err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48");
3679 @@ -101,6 +119,7 @@ int main(int argc, char** argv) {
3684 // Adding network to the tree
3685 err = loc_network_tree_add_network(tree, network2);
3687 @@ -117,20 +136,84 @@ int main(int argc, char** argv) {
3689 size_t nodes = loc_network_tree_count_nodes(tree);
3690 printf("The tree has %zu nodes\n", nodes);
3693 + // Check equals function
3694 + err = loc_network_cmp(network1, network1);
3696 + fprintf(stderr, "Network is not equal with itself\n");
3697 + exit(EXIT_FAILURE);
3700 + err = loc_network_cmp(network1, network2);
3702 + fprintf(stderr, "Networks equal unexpectedly\n");
3703 + exit(EXIT_FAILURE);
3706 // Check subnet function
3707 - err = loc_network_is_subnet_of(network1, network2);
3709 + err = loc_network_is_subnet(network1, network2);
3711 fprintf(stderr, "Subnet check 1 failed: %d\n", err);
3715 - err = loc_network_is_subnet_of(network2, network1);
3717 + err = loc_network_is_subnet(network2, network1);
3719 fprintf(stderr, "Subnet check 2 failed: %d\n", err);
3724 + struct loc_network* subnet1 = NULL;
3725 + struct loc_network* subnet2 = NULL;
3727 + err = loc_network_subnets(network1, &subnet1, &subnet2);
3728 + if (err || !subnet1 || !subnet2) {
3729 + fprintf(stderr, "Could not find subnets of network: %d\n", err);
3730 + exit(EXIT_FAILURE);
3733 + char* s = loc_network_str(subnet1);
3734 + printf("Received subnet1 = %s\n", s);
3737 + s = loc_network_str(subnet2);
3738 + printf("Received subnet2 = %s\n", s);
3741 + if (!loc_network_is_subnet(network1, subnet1)) {
3742 + fprintf(stderr, "Subnet1 is not a subnet\n");
3743 + exit(EXIT_FAILURE);
3746 + if (!loc_network_is_subnet(network1, subnet2)) {
3747 + fprintf(stderr, "Subnet2 is not a subnet\n");
3748 + exit(EXIT_FAILURE);
3751 + if (!loc_network_overlaps(network1, subnet1)) {
3752 + fprintf(stderr, "Network1 does not seem to contain subnet1\n");
3753 + exit(EXIT_FAILURE);
3756 + if (!loc_network_overlaps(network1, subnet2)) {
3757 + fprintf(stderr, "Network1 does not seem to contain subnet2\n");
3758 + exit(EXIT_FAILURE);
3761 + loc_network_unref(subnet1);
3762 + loc_network_unref(subnet2);
3764 + struct loc_network_list* excluded = loc_network_exclude(network1, network2);
3766 + fprintf(stderr, "Could not create excluded list\n");
3767 + exit(EXIT_FAILURE);
3770 + loc_network_list_dump(excluded);
3771 + loc_network_list_unref(excluded);
3773 // Create a database
3774 struct loc_writer* writer;
3775 err = loc_writer_new(ctx, &writer, NULL, NULL);
3776 @@ -160,6 +243,28 @@ int main(int argc, char** argv) {
3778 loc_network_set_asn(network4, 1024);
3780 + // Try adding an invalid network
3781 + struct loc_network* network;
3782 + err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
3783 + if (err != -EINVAL) {
3784 + fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
3785 + exit(EXIT_FAILURE);
3788 + // Try adding a single address
3789 + err = loc_writer_add_network(writer, &network, "2001:db8::");
3791 + fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
3792 + exit(EXIT_FAILURE);
3795 + // Try adding localhost
3796 + err = loc_writer_add_network(writer, &network, "::1/128");
3797 + if (err != -EINVAL) {
3798 + fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
3799 + exit(EXIT_FAILURE);
3802 FILE* f = tmpfile();
3804 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
3805 @@ -177,7 +282,10 @@ int main(int argc, char** argv) {
3806 loc_network_unref(network2);
3807 loc_network_unref(network3);
3808 loc_network_unref(network4);
3811 loc_network_tree_unref(tree);
3814 // And open it again from disk
3815 struct loc_database* db;
3816 diff --git a/src/writer.c b/src/writer.c
3817 index 5939cff..c61a6df 100644
3820 @@ -147,8 +147,19 @@ static void loc_writer_free(struct loc_writer* writer) {
3821 EVP_PKEY_free(writer->private_key2);
3824 - for (unsigned int i = 0; i < writer->as_count; i++) {
3825 - loc_as_unref(writer->as[i]);
3827 + for (unsigned int i = 0; i < writer->as_count; i++) {
3828 + loc_as_unref(writer->as[i]);
3833 + // Unref all countries
3834 + if (writer->countries) {
3835 + for (unsigned int i = 0; i < writer->countries_count; i++) {
3836 + loc_country_unref(writer->countries[i]);
3838 + free(writer->countries);
3841 // Release network tree
3842 @@ -601,7 +612,7 @@ static int loc_writer_create_signature(struct loc_writer* writer,
3846 - DEBUG(writer->ctx, "Successfully generated signature of %lu bytes\n", *length);
3847 + DEBUG(writer->ctx, "Successfully generated signature of %zu bytes\n", *length);