]> git.ipfire.org Git - people/pmueller/ipfire-2.x.git/blame - src/patches/libloc-0.9.4-upstream.patch
projects / people / pmueller / ipfire-2.x.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
libloc: Import latest fixes from upstream
[people/pmueller/ipfire-2.x.git] / src / patches / libloc-0.9.4-upstream.patch
CommitLineData
b952a52b 1diff --git a/Makefile.am b/Makefile.am
4a388bc9 2index a0431a6..dc594f8 100644
b952a52b
MT		 3--- a/Makefile.am
4+++ b/Makefile.am
5@@ -91,11 +91,14 @@ EXTRA_DIST += \
6 pkginclude_HEADERS = \
7 src/loc/libloc.h \
8 src/loc/as.h \
9+ src/loc/as-list.h \
10 src/loc/compat.h \
11 src/loc/country.h \
12+ src/loc/country-list.h \
13 src/loc/database.h \
14 src/loc/format.h \
15 src/loc/network.h \
16+ src/loc/network-list.h \
17 src/loc/private.h \
18 src/loc/stringpool.h \
19 src/loc/resolv.h \
20@@ -107,9 +110,12 @@ lib_LTLIBRARIES = \
21 src_libloc_la_SOURCES = \
22 src/libloc.c \
23 src/as.c \
24+ src/as-list.c \
25 src/country.c \
26+ src/country-list.c \
27 src/database.c \
28 src/network.c \
29+ src/network-list.c \
30 src/resolv.c \
31 src/stringpool.c \
32 src/writer.c
33@@ -312,6 +318,7 @@ check_PROGRAMS = \
34 src/test-database \
35 src/test-as \
36 src/test-network \
37+ src/test-network-list \
38 src/test-country \
39 src/test-signature
05db64d0 40
b952a52b
MT		 41@@ -351,6 +358,15 @@ src_test_network_CFLAGS = \
42 src_test_network_LDADD = \
43 src/libloc.la
05db64d0 44
b952a52b
MT		 45+src_test_network_list_SOURCES = \
46+ src/test-network-list.c
05db64d0 47+
b952a52b
MT		 48+src_test_network_list_CFLAGS = \
49+ $(TESTS_CFLAGS)
05db64d0 50+
b952a52b
MT		 51+src_test_network_list_LDADD = \
52+ src/libloc.la
05db64d0 53+
b952a52b
MT		 54 src_test_stringpool_SOURCES = \
55 src/test-stringpool.c
56
4a388bc9
MT		 57@@ -390,7 +406,7 @@ MANPAGES_XML = $(patsubst %.txt,%.xml,$(MANPAGES_TXT))
58 .PHONY: man
59 man: $(MANPAGES) $(MANPAGES_HTML)
60
61-if ENABLE_MANPAGES
62+if ENABLE_MAN_PAGES
63 man_MANS = \
64 $(MANPAGES)
65 endif
b952a52b 66diff --git a/configure.ac b/configure.ac
4a388bc9 67index 2364dfd..9eb9012 100644
b952a52b
MT		 68--- a/configure.ac
69+++ b/configure.ac
70@@ -1,6 +1,6 @@
71 AC_PREREQ(2.60)
72 AC_INIT([libloc],
73- [0.9.4],
74+ [0.9.5],
75 [location@lists.ipfire.org],
76 [libloc],
77 [https://location.ipfire.org/])
4a388bc9
MT		 78@@ -43,16 +43,16 @@ AC_PROG_MKDIR_P
79
80 # - man ------------------------------------------------------------------------
81
82-have_manpages=no
83-AC_ARG_ENABLE(manpages, AS_HELP_STRING([--disable-man-pages],
84+have_man_pages=no
85+AC_ARG_ENABLE(man_pages, AS_HELP_STRING([--disable-man-pages],
86 [do not install man pages]))
87-AS_IF([test "x$enable_manpages" != xno], [have_manpages=yes])
88-AM_CONDITIONAL(ENABLE_MANPAGES, [test "x$have_manpages" = "xyes"])
89+AS_IF([test "x$enable_man_pages" != xno], [have_man_pages=yes])
90+AM_CONDITIONAL(ENABLE_MAN_PAGES, [test "x$have_man_pages" = "xyes"])
91
92 AC_PATH_PROG([XSLTPROC], [xsltproc])
93
94 AC_CHECK_PROGS(ASCIIDOC, [asciidoc])
95-if test "${have_manpages}" = "yes" && test -z "${ASCIIDOC}"; then
96+if test "${have_man_pages}" = "yes" && test -z "${ASCIIDOC}"; then
97 AC_MSG_ERROR([Required program 'asciidoc' not found])
98 fi
99 # - debug ----------------------------------------------------------------------
b952a52b
MT		 100diff --git a/src/.gitignore b/src/.gitignore
101index caf80b5..3ccbdb8 100644
102--- a/src/.gitignore
103+++ b/src/.gitignore
104@@ -10,5 +10,6 @@ test-libloc
105 test-database
106 test-country
107 test-network
108+test-network-list
109 test-signature
110 test-stringpool
111diff --git a/src/as-list.c b/src/as-list.c
112new file mode 100644
113index 0000000..5acbb8a
114--- /dev/null
115+++ b/src/as-list.c
116@@ -0,0 +1,161 @@
117+/*
118+ libloc - A library to determine the location of someone on the Internet
05db64d0 119+
b952a52b 120+ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
05db64d0 121+
b952a52b
MT		 122+ This library is free software; you can redistribute it and/or
123+ modify it under the terms of the GNU Lesser General Public
124+ License as published by the Free Software Foundation; either
125+ version 2.1 of the License, or (at your option) any later version.
05db64d0 126+
b952a52b
MT		 127+ This library is distributed in the hope that it will be useful,
128+ but WITHOUT ANY WARRANTY; without even the implied warranty of
129+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
130+ Lesser General Public License for more details.
131+*/
05db64d0 132+
b952a52b
MT		 133+#include <errno.h>
134+#include <stdlib.h>
05db64d0 135+
b952a52b
MT		 136+#include <loc/as.h>
137+#include <loc/as-list.h>
138+#include <loc/private.h>
05db64d0 139+
b952a52b
MT		 140+struct loc_as_list {
141+ struct loc_ctx* ctx;
142+ int refcount;
05db64d0 143+
b952a52b
MT		 144+ struct loc_as** elements;
145+ size_t elements_size;
05db64d0 146+
b952a52b
MT		 147+ size_t size;
148+};
05db64d0 149+
b952a52b
MT		 150+static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
151+ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
152+ list, size, list->elements_size + size);
05db64d0 153+
b952a52b
MT		 154+ struct loc_as** elements = reallocarray(list->elements,
155+ list->elements_size + size, sizeof(*list->elements));
156+ if (!elements)
157+ return -errno;
05db64d0 158+
b952a52b
MT		 159+ list->elements = elements;
160+ list->elements_size += size;
05db64d0 161+
b952a52b
MT		 162+ return 0;
163+}
05db64d0 164+
b952a52b
MT		 165+LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
166+ struct loc_as_list** list) {
167+ struct loc_as_list* l = calloc(1, sizeof(*l));
168+ if (!l)
169+ return -ENOMEM;
05db64d0 170+
b952a52b
MT		 171+ l->ctx = loc_ref(ctx);
172+ l->refcount = 1;
05db64d0 173+
b952a52b
MT		 174+ DEBUG(l->ctx, "AS list allocated at %p\n", l);
175+ *list = l;
05db64d0 176+
b952a52b
MT		 177+ return 0;
178+}
05db64d0 179+
b952a52b
MT		 180+LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
181+ list->refcount++;
05db64d0 182+
b952a52b
MT		 183+ return list;
184+}
05db64d0 185+
b952a52b
MT		 186+static void loc_as_list_free(struct loc_as_list* list) {
187+ DEBUG(list->ctx, "Releasing AS list at %p\n", list);
05db64d0 188+
b952a52b 189+ loc_as_list_clear(list);
05db64d0 190+
b952a52b
MT		 191+ loc_unref(list->ctx);
192+ free(list);
193+}
05db64d0 194+
b952a52b
MT		 195+LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
196+ if (!list)
197+ return NULL;
198+
199+ if (--list->refcount > 0)
200+ return list;
201+
202+ loc_as_list_free(list);
203+ return NULL;
204+}
205+
206+LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
207+ return list->size;
208+}
209+
210+LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
211+ return list->size == 0;
212+}
213+
214+LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
215+ if (!list->elements)
216+ return;
217+
218+ for (unsigned int i = 0; i < list->size; i++)
219+ loc_as_unref(list->elements[i]);
220+
221+ free(list->elements);
222+ list->elements = NULL;
223+ list->elements_size = 0;
224+
225+ list->size = 0;
226+}
227+
228+LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
229+ // Check index
230+ if (index >= list->size)
231+ return NULL;
232+
233+ return loc_as_ref(list->elements[index]);
234+}
235+
236+LOC_EXPORT int loc_as_list_append(
237+ struct loc_as_list* list, struct loc_as* as) {
238+ if (loc_as_list_contains(list, as))
239+ return 0;
240+
241+ // Check if we have space left
242+ if (list->size >= list->elements_size) {
243+ int r = loc_as_list_grow(list, 64);
244+ if (r)
245+ return r;
246+ }
247+
248+ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
249+
250+ list->elements[list->size++] = loc_as_ref(as);
251+
252+ return 0;
253+}
254+
255+LOC_EXPORT int loc_as_list_contains(
256+ struct loc_as_list* list, struct loc_as* as) {
257+ for (unsigned int i = 0; i < list->size; i++) {
258+ if (loc_as_cmp(as, list->elements[i]) == 0)
259+ return 1;
260+ }
261+
262+ return 0;
263+}
264+
265+LOC_EXPORT int loc_as_list_contains_number(
266+ struct loc_as_list* list, uint32_t number) {
267+ struct loc_as* as;
268+
269+ int r = loc_as_new(list->ctx, &as, number);
270+ if (r)
271+ return -1;
272+
273+ r = loc_as_list_contains(list, as);
274+ loc_as_unref(as);
275+
276+ return r;
277+}
05db64d0 278diff --git a/src/as.c b/src/as.c
b952a52b 279index e1fbb01..757bf3d 100644
05db64d0
MT		 280--- a/src/as.c
281+++ b/src/as.c
282@@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
283 }
284
285 LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) {
286- as->name = strdup(name);
287+ if (as->name)
288+ free(as->name);
289+
290+ if (name)
291+ as->name = strdup(name);
292+ else
293+ as->name = NULL;
294
295 return 0;
296 }
b952a52b 297@@ -139,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
05db64d0
MT		 298 if (!string)
299 return 1;
300
301+ // Cannot match anything when name is not set
302+ if (!as->name)
303+ return 1;
304+
305 // Search if string is in name
306 if (strcasestr(as->name, string) != NULL)
307 return 1;
b952a52b
MT		 308diff --git a/src/country-list.c b/src/country-list.c
309new file mode 100644
310index 0000000..cc36740
311--- /dev/null
312+++ b/src/country-list.c
313@@ -0,0 +1,161 @@
314+/*
315+ libloc - A library to determine the location of someone on the Internet
05db64d0 316+
b952a52b 317+ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
05db64d0 318+
b952a52b
MT		 319+ This library is free software; you can redistribute it and/or
320+ modify it under the terms of the GNU Lesser General Public
321+ License as published by the Free Software Foundation; either
322+ version 2.1 of the License, or (at your option) any later version.
05db64d0 323+
b952a52b
MT		 324+ This library is distributed in the hope that it will be useful,
325+ but WITHOUT ANY WARRANTY; without even the implied warranty of
326+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
327+ Lesser General Public License for more details.
328+*/
05db64d0 329+
b952a52b
MT		 330+#include <errno.h>
331+#include <stdlib.h>
05db64d0 332+
b952a52b
MT		 333+#include <loc/country.h>
334+#include <loc/country-list.h>
335+#include <loc/private.h>
05db64d0 336+
b952a52b
MT		 337+struct loc_country_list {
338+ struct loc_ctx* ctx;
339+ int refcount;
05db64d0 340+
b952a52b
MT		 341+ struct loc_country** elements;
342+ size_t elements_size;
05db64d0 343+
b952a52b
MT		 344+ size_t size;
345+};
05db64d0 346+
b952a52b
MT		 347+static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
348+ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
349+ list, size, list->elements_size + size);
05db64d0 350+
b952a52b
MT		 351+ struct loc_country** elements = reallocarray(list->elements,
352+ list->elements_size + size, sizeof(*list->elements));
353+ if (!elements)
354+ return -errno;
05db64d0 355+
b952a52b
MT		 356+ list->elements = elements;
357+ list->elements_size += size;
05db64d0 358+
b952a52b
MT		 359+ return 0;
360+}
05db64d0 361+
b952a52b
MT		 362+LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
363+ struct loc_country_list** list) {
364+ struct loc_country_list* l = calloc(1, sizeof(*l));
365+ if (!l)
366+ return -ENOMEM;
05db64d0 367+
b952a52b
MT		 368+ l->ctx = loc_ref(ctx);
369+ l->refcount = 1;
05db64d0 370+
b952a52b
MT		 371+ DEBUG(l->ctx, "Country list allocated at %p\n", l);
372+ *list = l;
05db64d0 373+
b952a52b
MT		 374+ return 0;
375+}
05db64d0 376+
b952a52b
MT		 377+LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
378+ list->refcount++;
05db64d0 379+
b952a52b
MT		 380+ return list;
381+}
05db64d0 382+
b952a52b
MT		 383+static void loc_country_list_free(struct loc_country_list* list) {
384+ DEBUG(list->ctx, "Releasing country list at %p\n", list);
05db64d0 385+
b952a52b 386+ loc_country_list_clear(list);
05db64d0 387+
b952a52b
MT		 388+ loc_unref(list->ctx);
389+ free(list);
390+}
05db64d0 391+
b952a52b
MT		 392+LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
393+ if (!list)
394+ return NULL;
05db64d0 395+
b952a52b
MT		 396+ if (--list->refcount > 0)
397+ return list;
05db64d0 398+
b952a52b
MT		 399+ loc_country_list_free(list);
400+ return NULL;
401+}
05db64d0 402+
b952a52b
MT		 403+LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
404+ return list->size;
405+}
05db64d0 406+
b952a52b
MT		 407+LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
408+ return list->size == 0;
409+}
05db64d0 410+
b952a52b
MT		 411+LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
412+ if (!list->elements)
413+ return;
05db64d0 414+
b952a52b
MT		 415+ for (unsigned int i = 0; i < list->size; i++)
416+ loc_country_unref(list->elements[i]);
417+
418+ free(list->elements);
419+ list->elements = NULL;
420+ list->elements_size = 0;
421+
422+ list->size = 0;
05db64d0
MT		 423+}
424+
b952a52b
MT		 425+LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
426+ // Check index
427+ if (index >= list->size)
428+ return NULL;
05db64d0 429+
b952a52b
MT		 430+ return loc_country_ref(list->elements[index]);
431+}
05db64d0 432+
b952a52b
MT		 433+LOC_EXPORT int loc_country_list_append(
434+ struct loc_country_list* list, struct loc_country* country) {
435+ if (loc_country_list_contains(list, country))
436+ return 0;
05db64d0 437+
b952a52b
MT		 438+ // Check if we have space left
439+ if (list->size >= list->elements_size) {
440+ int r = loc_country_list_grow(list, 64);
441+ if (r)
442+ return r;
05db64d0
MT		 443+ }
444+
b952a52b 445+ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
05db64d0 446+
b952a52b 447+ list->elements[list->size++] = loc_country_ref(country);
05db64d0 448+
b952a52b
MT		 449+ return 0;
450+}
05db64d0 451+
b952a52b
MT		 452+LOC_EXPORT int loc_country_list_contains(
453+ struct loc_country_list* list, struct loc_country* country) {
454+ for (unsigned int i = 0; i < list->size; i++) {
455+ if (loc_country_cmp(country, list->elements[i]) == 0)
456+ return 1;
457+ }
05db64d0 458+
05db64d0
MT		 459+ return 0;
460+}
461+
b952a52b
MT		 462+LOC_EXPORT int loc_country_list_contains_code(
463+ struct loc_country_list* list, const char* code) {
464+ struct loc_country* country;
05db64d0 465+
b952a52b
MT		 466+ int r = loc_country_new(list->ctx, &country, code);
467+ if (r)
468+ return -1;
05db64d0 469+
b952a52b
MT		 470+ r = loc_country_list_contains(list, country);
471+ loc_country_unref(country);
05db64d0 472+
b952a52b
MT		 473+ return r;
474+}
475diff --git a/src/country.c b/src/country.c
476index 2ba93e6..7aac0db 100644
477--- a/src/country.c
478+++ b/src/country.c
479@@ -34,6 +34,9 @@ struct loc_country {
480 };
05db64d0 481
b952a52b
MT		 482 LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
483+ if (!loc_country_code_is_valid(country_code))
484+ return -EINVAL;
485+
486 struct loc_country* c = calloc(1, sizeof(*c));
487 if (!c)
488 return -ENOMEM;
05db64d0 489diff --git a/src/database.c b/src/database.c
460741fd 490index fa1dad0..4b8bf1d 100644
05db64d0
MT		 491--- a/src/database.c
492+++ b/src/database.c
b952a52b 493@@ -38,8 +38,10 @@
05db64d0 494
b952a52b
MT		 495 #include <loc/libloc.h>
496 #include <loc/as.h>
497+#include <loc/as-list.h>
498 #include <loc/compat.h>
499 #include <loc/country.h>
500+#include <loc/country-list.h>
501 #include <loc/database.h>
502 #include <loc/format.h>
503 #include <loc/network.h>
504@@ -99,11 +101,14 @@ struct loc_database_enumerator {
05db64d0 505
b952a52b
MT		 506 // Search string
507 char* string;
508- char country_code[3];
509- uint32_t asn;
510+ struct loc_country_list* countries;
511+ struct loc_as_list* asns;
512 enum loc_network_flags flags;
513 int family;
05db64d0 514
b952a52b
MT		 515+ // Flatten output?
516+ int flatten;
517+
518 // Index of the AS we are looking at
519 unsigned int as_index;
05db64d0 520
b952a52b
MT		 521@@ -115,6 +120,9 @@ struct loc_database_enumerator {
522 struct loc_node_stack network_stack[MAX_STACK_DEPTH];
523 int network_stack_depth;
524 unsigned int* networks_visited;
05db64d0 525+
b952a52b
MT		 526+ // For subnet search
527+ struct loc_network_list* stack;
528 };
05db64d0 529
b952a52b 530 static int loc_database_read_magic(struct loc_database* db) {
4a388bc9
MT		 531@@ -242,11 +250,11 @@ static int loc_database_read_signature(struct loc_database* db,
532 char** dst, char* src, size_t length) {
533 // Check for a plausible signature length
534 if (length > LOC_SIGNATURE_MAX_LENGTH) {
535- ERROR(db->ctx, "Signature too long: %ld\n", length);
536+ ERROR(db->ctx, "Signature too long: %zu\n", length);
537 return -EINVAL;
538 }
539
540- DEBUG(db->ctx, "Reading signature of %ld bytes\n", length);
541+ DEBUG(db->ctx, "Reading signature of %zu bytes\n", length);
542
543 // Allocate space
544 *dst = malloc(length);
b952a52b 545@@ -611,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
05db64d0
MT		 546 }
547
b952a52b
MT		 548 clock_t end = clock();
549- DEBUG(db->ctx, "Signature checked in %.4fms\n",
550+ INFO(db->ctx, "Signature checked in %.4fms\n",
551 (double)(end - start) / CLOCKS_PER_SEC * 1000);
05db64d0 552
b952a52b
MT		 553 CLEANUP:
554@@ -671,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
555 off_t lo = 0;
556 off_t hi = db->as_count - 1;
05db64d0 557
b952a52b
MT		 558+#ifdef ENABLE_DEBUG
559 // Save start time
560 clock_t start = clock();
561+#endif
05db64d0 562
b952a52b
MT		 563 while (lo <= hi) {
564 off_t i = (lo + hi) / 2;
565@@ -685,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
566 // Check if this is a match
567 uint32_t as_number = loc_as_get_number(*as);
568 if (as_number == number) {
569+#ifdef ENABLE_DEBUG
570 clock_t end = clock();
05db64d0 571
b952a52b
MT		 572 // Log how fast this has been
573 DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
574 (double)(end - start) / CLOCKS_PER_SEC * 1000);
575+#endif
05db64d0 576
b952a52b
MT		 577 return 0;
578 }
579@@ -733,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
580 return -1;
581 }
05db64d0 582
b952a52b
MT		 583+#ifdef ENABLE_DEBUG
584 if (r == 0) {
585 char* string = loc_network_str(*network);
586 DEBUG(db->ctx, "Got network %s\n", string);
587 free(string);
588 }
589+#endif
05db64d0 590
b952a52b 591 return r;
05db64d0 592 }
b952a52b
MT		 593@@ -762,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru
594 }
05db64d0 595
b952a52b
MT		 596 // Check if the given IP address is inside the network
597- r = loc_network_match_address(*network, address);
598- if (r) {
599+ if (!loc_network_match_address(*network, address)) {
600 DEBUG(db->ctx, "Searched address is not part of the network\n");
05db64d0 601
b952a52b
MT		 602 loc_network_unref(*network);
603@@ -832,17 +845,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
05db64d0 604
b952a52b 605 *network = NULL;
05db64d0 606
b952a52b
MT		 607+#ifdef ENABLE_DEBUG
608 // Save start time
609 clock_t start = clock();
610+#endif
05db64d0 611
b952a52b
MT		 612 int r = __loc_database_lookup(db, address, network, &network_address,
613 db->network_nodes_v1, 0);
05db64d0 614
b952a52b
MT		 615+#ifdef ENABLE_DEBUG
616 clock_t end = clock();
05db64d0 617
b952a52b
MT		 618 // Log how fast this has been
619 DEBUG(db->ctx, "Executed network search in %.4fms\n",
620 (double)(end - start) / CLOCKS_PER_SEC * 1000);
621+#endif
622
623 return r;
05db64d0 624 }
b952a52b
MT		 625@@ -889,8 +906,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
626 off_t lo = 0;
627 off_t hi = db->countries_count - 1;
05db64d0 628
b952a52b
MT		 629+#ifdef ENABLE_DEBUG
630 // Save start time
631 clock_t start = clock();
632+#endif
633
634 while (lo <= hi) {
635 off_t i = (lo + hi) / 2;
636@@ -905,11 +924,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
637 int result = strcmp(code, cc);
638
639 if (result == 0) {
640+#ifdef ENABLE_DEBUG
641 clock_t end = clock();
642
643 // Log how fast this has been
644 DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
645 (double)(end - start) / CLOCKS_PER_SEC * 1000);
646+#endif
647
648 return 0;
649 }
650@@ -932,8 +953,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
651
652 // Enumerator
653
654+static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
655+ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
656+
657+ // Release all references
658+ loc_database_unref(enumerator->db);
659+ loc_unref(enumerator->ctx);
660+
661+ if (enumerator->string)
662+ free(enumerator->string);
663+
664+ if (enumerator->countries)
665+ loc_country_list_unref(enumerator->countries);
666+
667+ if (enumerator->asns)
668+ loc_as_list_unref(enumerator->asns);
669+
670+ // Free network search
671+ free(enumerator->networks_visited);
672+
673+ // Free subnet stack
674+ if (enumerator->stack)
675+ loc_network_list_unref(enumerator->stack);
676+
677+ free(enumerator);
678+}
679+
680 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
681- struct loc_database* db, enum loc_database_enumerator_mode mode) {
682+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
683 struct loc_database_enumerator* e = calloc(1, sizeof(*e));
684 if (!e)
685 return -ENOMEM;
686@@ -944,11 +991,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
687 e->mode = mode;
688 e->refcount = 1;
689
690+ // Flatten output?
691+ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
692+
693 // Initialise graph search
694- //e->network_stack[++e->network_stack_depth] = 0;
695 e->network_stack_depth = 1;
696 e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
05db64d0 697
b952a52b
MT		 698+ // Allocate stack
699+ int r = loc_network_list_new(e->ctx, &e->stack);
700+ if (r) {
701+ loc_database_enumerator_free(e);
702+ return r;
703+ }
704+
705 DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
05db64d0 706
b952a52b
MT		 707 *enumerator = e;
708@@ -961,22 +1017,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
709 return enumerator;
05db64d0
MT		 710 }
711
b952a52b
MT		 712-static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
713- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
714-
715- // Release all references
716- loc_database_unref(enumerator->db);
717- loc_unref(enumerator->ctx);
718-
719- if (enumerator->string)
720- free(enumerator->string);
721-
722- // Free network search
723- free(enumerator->networks_visited);
724-
725- free(enumerator);
726-}
727-
728 LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
729 if (!enumerator)
730 return NULL;
731@@ -998,40 +1038,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
05db64d0
MT		 732 return 0;
733 }
734
b952a52b
MT		 735-LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
736- // Set empty country code
737- if (!country_code || !*country_code) {
738- *enumerator->country_code = '\0';
739- return 0;
740- }
741+LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
742+ struct loc_database_enumerator* enumerator) {
743+ if (!enumerator->countries)
744+ return NULL;
05db64d0 745
b952a52b
MT		 746- // Treat A1, A2, A3 as special country codes,
747- // but perform search for flags instead
748- if (strcmp(country_code, "A1") == 0) {
749- return loc_database_enumerator_set_flag(enumerator,
750- LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
751- } else if (strcmp(country_code, "A2") == 0) {
752- return loc_database_enumerator_set_flag(enumerator,
753- LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
754- } else if (strcmp(country_code, "A3") == 0) {
755- return loc_database_enumerator_set_flag(enumerator,
756- LOC_NETWORK_FLAG_ANYCAST);
757- }
758+ return loc_country_list_ref(enumerator->countries);
759+}
05db64d0 760
b952a52b
MT		 761- // Country codes must be two characters
762- if (!loc_country_code_is_valid(country_code))
763- return -EINVAL;
764+LOC_EXPORT int loc_database_enumerator_set_countries(
765+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
766+ if (enumerator->countries)
767+ loc_country_list_unref(enumerator->countries);
05db64d0 768
b952a52b
MT		 769- for (unsigned int i = 0; i < 3; i++) {
770- enumerator->country_code[i] = country_code[i];
771- }
772+ enumerator->countries = loc_country_list_ref(countries);
05db64d0 773
b952a52b 774 return 0;
05db64d0
MT		 775 }
776
b952a52b
MT		 777-LOC_EXPORT int loc_database_enumerator_set_asn(
778- struct loc_database_enumerator* enumerator, unsigned int asn) {
779- enumerator->asn = asn;
780+LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
781+ struct loc_database_enumerator* enumerator) {
782+ if (!enumerator->asns)
783+ return NULL;
784+
785+ return loc_as_list_ref(enumerator->asns);
786+}
787+
788+LOC_EXPORT int loc_database_enumerator_set_asns(
789+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
790+ if (enumerator->asns)
791+ loc_as_list_unref(enumerator->asns);
792+
793+ enumerator->asns = loc_as_list_ref(asns);
05db64d0 794
b952a52b
MT		 795 return 0;
796 }
797@@ -1110,16 +1148,64 @@ static int loc_database_enumerator_stack_push_node(
798 return 0;
05db64d0
MT		 799 }
800
b952a52b
MT		 801-LOC_EXPORT int loc_database_enumerator_next_network(
802- struct loc_database_enumerator* enumerator, struct loc_network** network) {
803- // Reset network
804- *network = NULL;
805+static int loc_database_enumerator_filter_network(
806+ struct loc_database_enumerator* enumerator, struct loc_network* network) {
807+ // Skip if the family does not match
808+ if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
809+ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
810+ return 1;
811+ }
05db64d0 812
b952a52b
MT		 813- // Do not do anything if not in network mode
814- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
815- return 0;
816+ // Skip if the country code does not match
817+ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
818+ const char* country_code = loc_network_get_country_code(network);
05db64d0 819
b952a52b
MT		 820- int r;
821+ if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
822+ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
823+ return 1;
824+ }
825+ }
826+
827+ // Skip if the ASN does not match
828+ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
829+ uint32_t asn = loc_network_get_asn(network);
830+
831+ if (!loc_as_list_contains_number(enumerator->asns, asn)) {
832+ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
833+ return 1;
834+ }
835+ }
836+
837+ // Skip if flags do not match
838+ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
839+ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
840+ return 1;
841+ }
842+
843+ // Do not filter
844+ return 0;
845+}
846+
847+static int __loc_database_enumerator_next_network(
848+ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
849+ // Return top element from the stack
850+ while (1) {
851+ *network = loc_network_list_pop_first(enumerator->stack);
852+
853+ // Stack is empty
854+ if (!*network)
855+ break;
856+
857+ // Throw away any networks by filter
858+ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
859+ loc_network_unref(*network);
860+ *network = NULL;
861+ continue;
862+ }
863+
864+ // Return result
865+ return 0;
866+ }
05db64d0 867
b952a52b
MT		 868 DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
869 enumerator->network_stack_depth);
870@@ -1149,7 +1235,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
871 enumerator->db->network_nodes_v1 + node->offset;
05db64d0 872
b952a52b
MT		 873 // Add edges to stack
874- r = loc_database_enumerator_stack_push_node(enumerator,
875+ int r = loc_database_enumerator_stack_push_node(enumerator,
876 be32toh(n->one), 1, node->depth + 1);
05db64d0 877
b952a52b 878 if (r)
460741fd 879@@ -1175,54 +1261,142 @@ LOC_EXPORT int loc_database_enumerator_next_network(
b952a52b
MT		 880 if (r)
881 return r;
05db64d0 882
b952a52b
MT		 883- // Check if we are interested in this network
884+ // Return all networks when the filter is disabled
885+ if (!filter)
886+ return 0;
05db64d0 887
b952a52b
MT		 888- // Skip if the family does not match
889- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
890+ // Check if we are interested in this network
891+ if (loc_database_enumerator_filter_network(enumerator, *network)) {
892 loc_network_unref(*network);
893 *network = NULL;
05db64d0 894
b952a52b
MT		 895 continue;
896 }
05db64d0 897
b952a52b
MT		 898- // Skip if the country code does not match
899- if (*enumerator->country_code &&
900- !loc_network_match_country_code(*network, enumerator->country_code)) {
901- loc_network_unref(*network);
902- *network = NULL;
903+ return 0;
904+ }
905+ }
05db64d0 906
b952a52b
MT		 907- continue;
908- }
909+ // Reached the end of the search
910+ return 0;
911+}
05db64d0 912
b952a52b
MT		 913- // Skip if the ASN does not match
914- if (enumerator->asn &&
915- !loc_network_match_asn(*network, enumerator->asn)) {
916- loc_network_unref(*network);
917- *network = NULL;
918+static int __loc_database_enumerator_next_network_flattened(
919+ struct loc_database_enumerator* enumerator, struct loc_network** network) {
920+ // Fetch the next network
921+ int r = __loc_database_enumerator_next_network(enumerator, network, 1);
922+ if (r)
923+ return r;
05db64d0 924
b952a52b
MT		 925- continue;
926- }
927+ // End if we could not read another network
928+ if (!*network)
929+ return 0;
05db64d0 930
b952a52b
MT		 931- // Skip if flags do not match
932- if (enumerator->flags &&
933- !loc_network_match_flag(*network, enumerator->flags)) {
934- loc_network_unref(*network);
935- *network = NULL;
936+ struct loc_network* subnet = NULL;
937+ struct loc_network_list* subnets;
05db64d0 938
b952a52b
MT		 939- continue;
940+ // Create a list with all subnets
941+ r = loc_network_list_new(enumerator->ctx, &subnets);
942+ if (r)
943+ return r;
944+
945+ // Search all subnets from the database
946+ while (1) {
947+ // Fetch the next network in line
948+ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
949+ if (r) {
950+ loc_network_unref(subnet);
951+ loc_network_list_unref(subnets);
952+
953+ return r;
954+ }
955+
956+ // End if we did not receive another subnet
957+ if (!subnet)
958+ break;
959+
960+ // Collect all subnets in a list
961+ if (loc_network_is_subnet(*network, subnet)) {
962+ r = loc_network_list_push(subnets, subnet);
963+ if (r) {
964+ loc_network_unref(subnet);
965+ loc_network_list_unref(subnets);
966+
967+ return r;
968 }
05db64d0 969
b952a52b
MT		 970- return 0;
971+ loc_network_unref(subnet);
972+ continue;
460741fd 973+ }
b952a52b
MT		 974+
975+ // If this is not a subnet, we push it back onto the stack and break
976+ r = loc_network_list_push(enumerator->stack, subnet);
977+ if (r) {
978+ loc_network_unref(subnet);
979+ loc_network_list_unref(subnets);
980+
981+ return r;
460741fd 982 }
b952a52b
MT		 983+
984+ loc_network_unref(subnet);
985+ break;
986 }
05db64d0 987
b952a52b
MT		 988- // Reached the end of the search
989+ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
460741fd
MT		 990
991- // Mark all nodes as non-visited
992- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
993- enumerator->networks_visited[i] = 0;
b952a52b
MT		 994+ // We can abort here if the network has no subnets
995+ if (loc_network_list_empty(subnets)) {
996+ loc_network_list_unref(subnets);
460741fd
MT		 997
998- return 0;
b952a52b
MT		 999+ return 0;
1000+ }
460741fd 1001+
b952a52b
MT		 1002+ // If the network has any subnets, we will break it into smaller parts
1003+ // without the subnets.
1004+ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
1005+ if (!excluded) {
1006+ loc_network_list_unref(subnets);
1007+ return -1;
1008+ }
1009+
1010+ // Merge subnets onto the stack
1011+ r = loc_network_list_merge(enumerator->stack, subnets);
1012+ if (r) {
1013+ loc_network_list_unref(subnets);
1014+ loc_network_list_unref(excluded);
1015+
1016+ return r;
1017+ }
1018+
1019+ // Push excluded list onto the stack
1020+ r = loc_network_list_merge(enumerator->stack, excluded);
1021+ if (r) {
1022+ loc_network_list_unref(subnets);
1023+ loc_network_list_unref(excluded);
1024+
1025+ return r;
1026+ }
1027+
1028+ loc_network_list_unref(subnets);
1029+ loc_network_list_unref(excluded);
1030+
460741fd 1031+ // Drop the network and restart the whole process again to pick the next network
b952a52b 1032+ loc_network_unref(*network);
460741fd
MT		 1033+
1034+ return __loc_database_enumerator_next_network_flattened(enumerator, network);
1035+}
1036+
b952a52b
MT		 1037+LOC_EXPORT int loc_database_enumerator_next_network(
1038+ struct loc_database_enumerator* enumerator, struct loc_network** network) {
1039+ // Do not do anything if not in network mode
1040+ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
1041+ return 0;
1042+
1043+ // Flatten output?
1044+ if (enumerator->flatten)
1045+ return __loc_database_enumerator_next_network_flattened(enumerator, network);
1046+
1047+ return __loc_database_enumerator_next_network(enumerator, network, 1);
460741fd
MT		 1048 }
1049
b952a52b 1050 LOC_EXPORT int loc_database_enumerator_next_country(
05db64d0 1051diff --git a/src/libloc.sym b/src/libloc.sym
b952a52b 1052index b8296eb..ee333f1 100644
05db64d0
MT		 1053--- a/src/libloc.sym
1054+++ b/src/libloc.sym
b952a52b
MT		 1055@@ -37,6 +37,18 @@ global:
1056 loc_as_set_name;
1057 loc_as_unref;
1058
1059+ # AS List
1060+ loc_as_list_append;
1061+ loc_as_list_clear;
1062+ loc_as_list_contains;
1063+ loc_as_list_contains_number;
1064+ loc_as_list_empty;
1065+ loc_as_list_get;
1066+ loc_as_list_new;
1067+ loc_as_list_ref;
1068+ loc_as_list_size;
1069+ loc_as_list_unref;
1070+
1071 # Country
1072 loc_country_cmp;
1073 loc_country_code_is_valid;
1074@@ -49,6 +61,18 @@ global:
1075 loc_country_set_name;
1076 loc_country_unref;
1077
1078+ # Country List
1079+ loc_country_list_append;
1080+ loc_country_list_clear;
1081+ loc_country_list_contains;
1082+ loc_country_list_contains_code;
1083+ loc_country_list_empty;
1084+ loc_country_list_get;
1085+ loc_country_list_new;
1086+ loc_country_list_ref;
1087+ loc_country_list_size;
1088+ loc_country_list_unref;
1089+
1090 # Database
1091 loc_database_add_as;
1092 loc_database_count_as;
1093@@ -66,13 +90,15 @@ global:
1094 loc_database_verify;
1095
1096 # Database Enumerator
1097+ loc_database_enumerator_get_asns;
1098+ loc_database_enumerator_get_countries;
1099 loc_database_enumerator_new;
1100 loc_database_enumerator_next_as;
1101 loc_database_enumerator_next_country;
1102 loc_database_enumerator_next_network;
1103 loc_database_enumerator_ref;
1104- loc_database_enumerator_set_asn;
1105- loc_database_enumerator_set_country_code;
1106+ loc_database_enumerator_set_asns;
1107+ loc_database_enumerator_set_countries;
1108 loc_database_enumerator_set_family;
1109 loc_database_enumerator_set_flag;
1110 loc_database_enumerator_set_string;
1111@@ -80,24 +106,48 @@ global:
1112
1113 # Network
1114 loc_network_address_family;
1115+ loc_network_cmp;
1116+ loc_network_exclude;
1117+ loc_network_exclude_list;
1118 loc_network_format_first_address;
05db64d0
MT		 1119 loc_network_format_last_address;
1120 loc_network_get_asn;
1121 loc_network_get_country_code;
b952a52b
MT		 1122+ loc_network_get_first_address;
1123+ loc_network_get_last_address;
05db64d0 1124 loc_network_has_flag;
b952a52b
MT		 1125- loc_network_is_subnet_of;
1126+ loc_network_is_subnet;
1127+ loc_network_match_address;
1128 loc_network_match_asn;
1129 loc_network_match_country_code;
1130 loc_network_match_flag;
1131 loc_network_new;
1132 loc_network_new_from_string;
1133+ loc_network_overlaps;
1134+ loc_network_prefix;
1135 loc_network_ref;
1136 loc_network_set_asn;
1137 loc_network_set_country_code;
1138 loc_network_set_flag;
1139 loc_network_str;
1140+ loc_network_subnets;
1141 loc_network_unref;
1142
1143+ # Network List
1144+ loc_network_list_clear;
1145+ loc_network_list_contains;
1146+ loc_network_list_dump;
1147+ loc_network_list_empty;
1148+ loc_network_list_get;
1149+ loc_network_list_merge;
1150+ loc_network_list_new;
1151+ loc_network_list_pop;
1152+ loc_network_list_pop_first;
1153+ loc_network_list_push;
1154+ loc_network_list_ref;
1155+ loc_network_list_size;
1156+ loc_network_list_unref;
1157+
1158 # Writer
1159 loc_writer_add_as;
1160 loc_writer_add_country;
1161diff --git a/src/loc/as-list.h b/src/loc/as-list.h
05db64d0 1162new file mode 100644
b952a52b 1163index 0000000..7b5c4e8
05db64d0 1164--- /dev/null
b952a52b
MT		 1165+++ b/src/loc/as-list.h
1166@@ -0,0 +1,41 @@
05db64d0
MT		 1167+/*
1168+ libloc - A library to determine the location of someone on the Internet
1169+
b952a52b 1170+ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
05db64d0
MT		 1171+
1172+ This library is free software; you can redistribute it and/or
1173+ modify it under the terms of the GNU Lesser General Public
1174+ License as published by the Free Software Foundation; either
1175+ version 2.1 of the License, or (at your option) any later version.
1176+
1177+ This library is distributed in the hope that it will be useful,
1178+ but WITHOUT ANY WARRANTY; without even the implied warranty of
1179+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1180+ Lesser General Public License for more details.
1181+*/
1182+
b952a52b
MT		 1183+#ifndef LIBLOC_AS_LIST_H
1184+#define LIBLOC_AS_LIST_H
05db64d0 1185+
b952a52b
MT		 1186+#include <loc/as.h>
1187+#include <loc/libloc.h>
1188+
1189+struct loc_as_list;
1190+
1191+int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
1192+struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
1193+struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
1194+
1195+size_t loc_as_list_size(struct loc_as_list* list);
1196+int loc_as_list_empty(struct loc_as_list* list);
1197+void loc_as_list_clear(struct loc_as_list* list);
1198+
1199+struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
1200+int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
1201+
1202+int loc_as_list_contains(
1203+ struct loc_as_list* list, struct loc_as* as);
1204+int loc_as_list_contains_number(
1205+ struct loc_as_list* list, uint32_t number);
05db64d0
MT		 1206+
1207+#endif
b952a52b 1208diff --git a/src/loc/country-list.h b/src/loc/country-list.h
05db64d0 1209new file mode 100644
b952a52b 1210index 0000000..a7f818a
05db64d0 1211--- /dev/null
b952a52b
MT		 1212+++ b/src/loc/country-list.h
1213@@ -0,0 +1,43 @@
05db64d0
MT		 1214+/*
1215+ libloc - A library to determine the location of someone on the Internet
1216+
b952a52b 1217+ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
05db64d0
MT		 1218+
1219+ This library is free software; you can redistribute it and/or
1220+ modify it under the terms of the GNU Lesser General Public
1221+ License as published by the Free Software Foundation; either
1222+ version 2.1 of the License, or (at your option) any later version.
1223+
1224+ This library is distributed in the hope that it will be useful,
1225+ but WITHOUT ANY WARRANTY; without even the implied warranty of
b952a52b
MT		 1226+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1227+ Lesser General Public License for more details.
1228+*/
05db64d0 1229+
b952a52b
MT		 1230+#ifndef LIBLOC_COUNTRY_LIST_H
1231+#define LIBLOC_COUNTRY_LIST_H
05db64d0 1232+
b952a52b 1233+#include <stdlib.h>
05db64d0 1234+
b952a52b
MT		 1235+#include <loc/libloc.h>
1236+#include <loc/country.h>
05db64d0 1237+
b952a52b 1238+struct loc_country_list;
05db64d0 1239+
b952a52b
MT		 1240+int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
1241+struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
1242+struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
05db64d0 1243+
b952a52b
MT		 1244+size_t loc_country_list_size(struct loc_country_list* list);
1245+int loc_country_list_empty(struct loc_country_list* list);
1246+void loc_country_list_clear(struct loc_country_list* list);
05db64d0 1247+
b952a52b
MT		 1248+struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
1249+int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
05db64d0 1250+
b952a52b
MT		 1251+int loc_country_list_contains(
1252+ struct loc_country_list* list, struct loc_country* country);
1253+int loc_country_list_contains_code(
1254+ struct loc_country_list* list, const char* code);
05db64d0 1255+
b952a52b
MT		 1256+#endif
1257diff --git a/src/loc/database.h b/src/loc/database.h
1258index 43173dd..70801f0 100644
1259--- a/src/loc/database.h
1260+++ b/src/loc/database.h
1261@@ -25,6 +25,7 @@
1262 #include <loc/network.h>
1263 #include <loc/as.h>
1264 #include <loc/country.h>
1265+#include <loc/country-list.h>
1266
1267 struct loc_database;
1268 int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
1269@@ -55,15 +56,24 @@ enum loc_database_enumerator_mode {
1270 LOC_DB_ENUMERATE_COUNTRIES = 3,
1271 };
1272
1273+enum loc_database_enumerator_flags {
1274+ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
1275+};
05db64d0 1276+
b952a52b
MT		 1277 struct loc_database_enumerator;
1278 int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
1279- struct loc_database* db, enum loc_database_enumerator_mode mode);
1280+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
1281 struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
1282 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
1283
1284 int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
1285-int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
1286-int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
1287+struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
1288+int loc_database_enumerator_set_countries(
1289+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
1290+struct loc_as_list* loc_database_enumerator_get_asns(
1291+ struct loc_database_enumerator* enumerator);
1292+int loc_database_enumerator_set_asns(
1293+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
1294 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
1295 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
1296 int loc_database_enumerator_next_as(
1297diff --git a/src/loc/network-list.h b/src/loc/network-list.h
1298new file mode 100644
1299index 0000000..bee21c4
1300--- /dev/null
1301+++ b/src/loc/network-list.h
1302@@ -0,0 +1,37 @@
1303+/*
1304+ libloc - A library to determine the location of someone on the Internet
05db64d0 1305+
b952a52b 1306+ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
05db64d0 1307+
b952a52b
MT		 1308+ This library is free software; you can redistribute it and/or
1309+ modify it under the terms of the GNU Lesser General Public
1310+ License as published by the Free Software Foundation; either
1311+ version 2.1 of the License, or (at your option) any later version.
05db64d0 1312+
b952a52b
MT		 1313+ This library is distributed in the hope that it will be useful,
1314+ but WITHOUT ANY WARRANTY; without even the implied warranty of
1315+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1316+ Lesser General Public License for more details.
1317+*/
05db64d0 1318+
b952a52b
MT		 1319+#ifndef LIBLOC_NETWORK_LIST_H
1320+#define LIBLOC_NETWORK_LIST_H
05db64d0 1321+
b952a52b 1322+#include <loc/network.h>
05db64d0 1323+
b952a52b
MT		 1324+struct loc_network_list;
1325+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
1326+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
1327+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
1328+size_t loc_network_list_size(struct loc_network_list* list);
1329+int loc_network_list_empty(struct loc_network_list* list);
1330+void loc_network_list_clear(struct loc_network_list* list);
1331+void loc_network_list_dump(struct loc_network_list* list);
1332+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
1333+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
1334+struct loc_network* loc_network_list_pop(struct loc_network_list* list);
1335+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
1336+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
1337+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
05db64d0 1338+
b952a52b
MT		 1339+#endif
1340diff --git a/src/loc/network.h b/src/loc/network.h
1341index 70c3803..af3dafd 100644
1342--- a/src/loc/network.h
1343+++ b/src/loc/network.h
1344@@ -21,6 +21,7 @@
1345
1346 #include <loc/libloc.h>
1347 #include <loc/format.h>
05db64d0 1348+#include <loc/network-list.h>
05db64d0 1349
b952a52b
MT		 1350 enum loc_network_flags {
1351 LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
1352@@ -37,8 +38,11 @@ struct loc_network* loc_network_ref(struct loc_network* network);
1353 struct loc_network* loc_network_unref(struct loc_network* network);
1354 char* loc_network_str(struct loc_network* network);
1355 int loc_network_address_family(struct loc_network* network);
1356+unsigned int loc_network_prefix(struct loc_network* network);
05db64d0 1357
b952a52b
MT		 1358+const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
1359 char* loc_network_format_first_address(struct loc_network* network);
1360+const struct in6_addr* loc_network_get_last_address(struct loc_network* network);
1361 char* loc_network_format_last_address(struct loc_network* network);
1362 int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
05db64d0 1363
b952a52b
MT		 1364@@ -54,7 +58,14 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
1365 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
1366 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
1367
1368-int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
1369+int loc_network_cmp(struct loc_network* self, struct loc_network* other);
1370+int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
1371+int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
1372+int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
1373+struct loc_network_list* loc_network_exclude(
1374+ struct loc_network* self, struct loc_network* other);
1375+struct loc_network_list* loc_network_exclude_list(
1376+ struct loc_network* network, struct loc_network_list* list);
1377
1378 #ifdef LIBLOC_PRIVATE
05db64d0 1379
b952a52b 1380diff --git a/src/network-list.c b/src/network-list.c
05db64d0 1381new file mode 100644
4a388bc9 1382index 0000000..698d3ab
05db64d0 1383--- /dev/null
b952a52b 1384+++ b/src/network-list.c
4a388bc9 1385@@ -0,0 +1,299 @@
05db64d0
MT		 1386+/*
1387+ libloc - A library to determine the location of someone on the Internet
1388+
1389+ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
1390+
1391+ This library is free software; you can redistribute it and/or
1392+ modify it under the terms of the GNU Lesser General Public
1393+ License as published by the Free Software Foundation; either
1394+ version 2.1 of the License, or (at your option) any later version.
1395+
1396+ This library is distributed in the hope that it will be useful,
1397+ but WITHOUT ANY WARRANTY; without even the implied warranty of
1398+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1399+ Lesser General Public License for more details.
1400+*/
1401+
1402+#include <errno.h>
1403+#include <stdlib.h>
b952a52b 1404+#include <time.h>
05db64d0 1405+
b952a52b
MT		 1406+#include <loc/libloc.h>
1407+#include <loc/network.h>
05db64d0
MT		 1408+#include <loc/private.h>
1409+
b952a52b 1410+struct loc_network_list {
05db64d0
MT		 1411+ struct loc_ctx* ctx;
1412+ int refcount;
1413+
b952a52b
MT		 1414+ struct loc_network** elements;
1415+ size_t elements_size;
1416+
05db64d0 1417+ size_t size;
05db64d0
MT		 1418+};
1419+
b952a52b
MT		 1420+static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
1421+ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
1422+ list, size, list->elements_size + size);
1423+
1424+ struct loc_network** elements = reallocarray(list->elements,
1425+ list->elements_size + size, sizeof(*list->elements));
1426+ if (!elements)
1427+ return -errno;
1428+
1429+ list->elements = elements;
1430+ list->elements_size += size;
1431+
1432+ return 0;
1433+}
1434+
1435+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
1436+ struct loc_network_list** list) {
1437+ struct loc_network_list* l = calloc(1, sizeof(*l));
05db64d0
MT		 1438+ if (!l)
1439+ return -ENOMEM;
1440+
1441+ l->ctx = loc_ref(ctx);
1442+ l->refcount = 1;
1443+
b952a52b 1444+ DEBUG(l->ctx, "Network list allocated at %p\n", l);
05db64d0 1445+ *list = l;
05db64d0
MT		 1446+ return 0;
1447+}
1448+
b952a52b 1449+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
05db64d0
MT		 1450+ list->refcount++;
1451+
1452+ return list;
1453+}
1454+
b952a52b
MT		 1455+static void loc_network_list_free(struct loc_network_list* list) {
1456+ DEBUG(list->ctx, "Releasing network list at %p\n", list);
05db64d0 1457+
b952a52b
MT		 1458+ // Remove all content
1459+ loc_network_list_clear(list);
05db64d0
MT		 1460+
1461+ loc_unref(list->ctx);
1462+ free(list);
1463+}
1464+
b952a52b 1465+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
05db64d0
MT		 1466+ if (!list)
1467+ return NULL;
1468+
1469+ if (--list->refcount > 0)
1470+ return list;
1471+
b952a52b 1472+ loc_network_list_free(list);
05db64d0
MT		 1473+ return NULL;
1474+}
1475+
b952a52b 1476+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
05db64d0
MT		 1477+ return list->size;
1478+}
1479+
b952a52b 1480+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
05db64d0
MT		 1481+ return list->size == 0;
1482+}
1483+
b952a52b
MT		 1484+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
1485+ if (!list->elements)
1486+ return;
05db64d0 1487+
b952a52b
MT		 1488+ for (unsigned int i = 0; i < list->size; i++)
1489+ loc_network_unref(list->elements[i]);
05db64d0 1490+
b952a52b
MT		 1491+ free(list->elements);
1492+ list->elements = NULL;
1493+ list->elements_size = 0;
05db64d0 1494+
b952a52b
MT		 1495+ list->size = 0;
1496+}
05db64d0 1497+
b952a52b
MT		 1498+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
1499+ struct loc_network* network;
1500+ char* s;
05db64d0 1501+
b952a52b
MT		 1502+ for (unsigned int i = 0; i < list->size; i++) {
1503+ network = list->elements[i];
05db64d0 1504+
b952a52b 1505+ s = loc_network_str(network);
05db64d0 1506+
b952a52b
MT		 1507+ INFO(list->ctx, "%4d: %s\n", i, s);
1508+ free(s);
1509+ }
1510+}
05db64d0 1511+
b952a52b
MT		 1512+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
1513+ // Check index
1514+ if (index >= list->size)
1515+ return NULL;
05db64d0 1516+
b952a52b
MT		 1517+ return loc_network_ref(list->elements[index]);
1518+}
05db64d0 1519+
b952a52b
MT		 1520+static off_t loc_network_list_find(struct loc_network_list* list,
1521+ struct loc_network* network, int* found) {
4a388bc9
MT		 1522+ // Insert at the beginning for an empty list
1523+ if (loc_network_list_empty(list))
1524+ return 0;
1525+
b952a52b
MT		 1526+ off_t lo = 0;
1527+ off_t hi = list->size - 1;
1528+ int result;
05db64d0 1529+
b952a52b
MT		 1530+ // Since we are working on an ordered list, there is often a good chance that
1531+ // the network we are looking for is at the end or has to go to the end.
1532+ if (hi >= 0) {
1533+ result = loc_network_cmp(network, list->elements[hi]);
05db64d0 1534+
b952a52b
MT		 1535+ // Match, so we are done
1536+ if (result == 0) {
1537+ *found = 1;
05db64d0 1538+
b952a52b
MT		 1539+ return hi;
1540+
1541+ // This needs to be added after the last one
1542+ } else if (result > 0) {
1543+ *found = 0;
1544+
1545+ return hi + 1;
1546+ }
1547+ }
05db64d0 1548+
b952a52b
MT		 1549+#ifdef ENABLE_DEBUG
1550+ // Save start time
1551+ clock_t start = clock();
05db64d0 1552+#endif
05db64d0 1553+
b952a52b 1554+ off_t i = 0;
05db64d0 1555+
b952a52b
MT		 1556+ while (lo <= hi) {
1557+ i = (lo + hi) / 2;
05db64d0 1558+
b952a52b
MT		 1559+ // Check if this is a match
1560+ result = loc_network_cmp(network, list->elements[i]);
05db64d0 1561+
b952a52b
MT		 1562+ if (result == 0) {
1563+ *found = 1;
05db64d0 1564+
b952a52b
MT		 1565+#ifdef ENABLE_DEBUG
1566+ clock_t end = clock();
05db64d0 1567+
b952a52b
MT		 1568+ // Log how fast this has been
1569+ DEBUG(list->ctx, "Found network in %.4fms at %jd\n",
1570+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
1571+#endif
05db64d0 1572+
b952a52b
MT		 1573+ return i;
1574+ }
05db64d0 1575+
b952a52b
MT		 1576+ if (result > 0) {
1577+ lo = i + 1;
1578+ i++;
1579+ } else {
1580+ hi = i - 1;
05db64d0 1581+ }
b952a52b 1582+ }
05db64d0 1583+
b952a52b 1584+ *found = 0;
05db64d0 1585+
b952a52b
MT		 1586+#ifdef ENABLE_DEBUG
1587+ clock_t end = clock();
05db64d0 1588+
b952a52b
MT		 1589+ // Log how fast this has been
1590+ DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n",
1591+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
1592+#endif
05db64d0 1593+
b952a52b
MT		 1594+ return i;
1595+}
1596+
1597+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
1598+ int found = 0;
1599+
1600+ off_t index = loc_network_list_find(list, network, &found);
1601+
1602+ // The network has been found on the list. Nothing to do.
1603+ if (found)
1604+ return 0;
1605+
1606+ DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n",
1607+ list, network, (intmax_t)index);
05db64d0 1608+
b952a52b
MT		 1609+ // Check if we have space left
1610+ if (list->size >= list->elements_size) {
1611+ int r = loc_network_list_grow(list, 64);
1612+ if (r)
1613+ return r;
1614+ }
05db64d0 1615+
b952a52b
MT		 1616+ // The list is now larger
1617+ list->size++;
05db64d0 1618+
b952a52b
MT		 1619+ // Move all elements out of the way
1620+ for (unsigned int i = list->size - 1; i > index; i--)
1621+ list->elements[i] = list->elements[i - 1];
05db64d0 1622+
b952a52b
MT		 1623+ // Add the new element at the right place
1624+ list->elements[index] = loc_network_ref(network);
05db64d0 1625+
b952a52b
MT		 1626+ return 0;
1627+}
05db64d0 1628+
b952a52b
MT		 1629+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
1630+ // Return nothing when empty
1631+ if (loc_network_list_empty(list)) {
1632+ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
1633+ return NULL;
1634+ }
05db64d0 1635+
b952a52b 1636+ struct loc_network* network = list->elements[--list->size];
05db64d0 1637+
b952a52b 1638+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
05db64d0 1639+
b952a52b
MT		 1640+ return network;
1641+}
1642+
1643+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
1644+ // Return nothing when empty
1645+ if (loc_network_list_empty(list)) {
1646+ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
1647+ return NULL;
1648+ }
05db64d0 1649+
b952a52b 1650+ struct loc_network* network = list->elements[0];
05db64d0 1651+
b952a52b
MT		 1652+ // Move all elements to the top of the stack
1653+ for (unsigned int i = 0; i < list->size - 1; i++) {
1654+ list->elements[i] = list->elements[i+1];
1655+ }
1656+
1657+ // The list is shorter now
1658+ --list->size;
1659+
1660+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
1661+
1662+ return network;
1663+}
1664+
1665+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
1666+ int found = 0;
1667+
1668+ loc_network_list_find(list, network, &found);
1669+
1670+ return found;
1671+}
1672+
1673+LOC_EXPORT int loc_network_list_merge(
1674+ struct loc_network_list* self, struct loc_network_list* other) {
1675+ int r;
1676+
1677+ for (unsigned int i = 0; i < other->size; i++) {
1678+ r = loc_network_list_push(self, other->elements[i]);
1679+ if (r)
1680+ return r;
1681+ }
05db64d0
MT		 1682+
1683+ return 0;
1684+}
b952a52b
MT		 1685diff --git a/src/network.c b/src/network.c
1686index 366caa2..a6b679c 100644
1687--- a/src/network.c
1688+++ b/src/network.c
1689@@ -29,6 +29,7 @@
1690 #include <loc/compat.h>
1691 #include <loc/country.h>
1692 #include <loc/network.h>
1693+#include <loc/network-list.h>
1694 #include <loc/private.h>
1695
1696 struct loc_network {
1697@@ -97,6 +98,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
1698 return a;
1699 }
1700
1701+static struct in6_addr address_increment(const struct in6_addr* address) {
1702+ struct in6_addr a = *address;
05db64d0 1703+
b952a52b
MT		 1704+ for (int octet = 15; octet >= 0; octet--) {
1705+ if (a.s6_addr[octet] < 255) {
1706+ a.s6_addr[octet]++;
1707+ break;
1708+ } else {
1709+ a.s6_addr[octet] = 0;
1710+ }
1711+ }
05db64d0 1712+
b952a52b 1713+ return a;
05db64d0
MT		 1714+}
1715+
b952a52b
MT		 1716 LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
1717 struct in6_addr* address, unsigned int prefix) {
1718 // Address cannot be unspecified
1719@@ -160,9 +176,11 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
1720 LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
1721 const char* address_string) {
1722 struct in6_addr first_address;
1723- unsigned int prefix = 0;
1724 char* prefix_string;
1725- int r = 1;
1726+ unsigned int prefix = 128;
1727+ int r = -EINVAL;
05db64d0 1728+
b952a52b
MT		 1729+ DEBUG(ctx, "Attempting to parse network %s\n", address_string);
1730
1731 // Make a copy of the string to work on it
1732 char* buffer = strdup(address_string);
1733@@ -171,29 +189,40 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1734 // Split address and prefix
1735 address_string = strsep(&prefix_string, "/");
1736
1737- // Did we find a prefix?
1738+ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
05db64d0 1739+
b952a52b
MT		 1740+ // Parse the address
1741+ r = loc_parse_address(ctx, address_string, &first_address);
1742+ if (r) {
1743+ DEBUG(ctx, "The address could not be parsed\n");
1744+ goto FAIL;
1745+ }
1746+
1747+ // If a prefix was given, we will try to parse it
1748 if (prefix_string) {
1749 // Convert prefix to integer
1750 prefix = strtol(prefix_string, NULL, 10);
1751
1752- if (prefix) {
1753- // Parse the address
1754- r = loc_parse_address(ctx, address_string, &first_address);
1755-
1756- // Map the prefix to IPv6 if needed
1757- if (IN6_IS_ADDR_V4MAPPED(&first_address))
1758- prefix += 96;
1759+ if (!prefix) {
1760+ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
1761+ goto FAIL;
1762 }
1763+
1764+ // Map the prefix to IPv6 if needed
1765+ if (IN6_IS_ADDR_V4MAPPED(&first_address))
1766+ prefix += 96;
1767 }
1768
1769+FAIL:
1770 // Free temporary buffer
1771 free(buffer);
1772
1773- if (r == 0) {
1774- r = loc_network_new(ctx, network, &first_address, prefix);
1775- }
1776+ // Exit if the parsing was unsuccessful
1777+ if (r)
1778+ return r;
1779
1780- return r;
1781+ // Create a new network
1782+ return loc_network_new(ctx, network, &first_address, prefix);
1783 }
1784
1785 LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
1786@@ -281,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) {
1787 return network->family;
1788 }
1789
1790+LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) {
1791+ switch (network->family) {
1792+ case AF_INET6:
1793+ return network->prefix;
1794+
1795+ case AF_INET:
1796+ return network->prefix - 96;
1797+ }
1798+
1799+ return 0;
05db64d0
MT		 1800+}
1801+
b952a52b
MT		 1802 static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) {
1803 const size_t length = INET6_ADDRSTRLEN;
1804
1805@@ -314,10 +355,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc
1806 return string;
1807 }
1808
1809+LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) {
1810+ return &network->first_address;
1811+}
05db64d0 1812+
b952a52b
MT		 1813 LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) {
1814 return loc_network_format_address(network, &network->first_address);
1815 }
1816
1817+LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) {
1818+ return &network->last_address;
1819+}
1820+
1821 LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
1822 return loc_network_format_address(network, &network->last_address);
1823 }
1824@@ -325,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
1825 LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) {
1826 // Address must be larger than the start address
1827 if (in6_addr_cmp(&network->first_address, address) > 0)
1828- return 1;
1829+ return 0;
1830
1831 // Address must be smaller than the last address
1832 if (in6_addr_cmp(&network->last_address, address) < 0)
1833- return 1;
1834+ return 0;
1835
1836 // The address is inside this network
1837- return 0;
1838+ return 1;
1839 }
1840
1841 LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) {
1842@@ -392,20 +441,310 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
1843 return loc_network_has_flag(network, flag);
1844 }
1845
1846-LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
1847+LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
1848+ // Compare address
1849+ int r = in6_addr_cmp(&self->first_address, &other->first_address);
1850+ if (r)
1851+ return r;
1852+
1853+ // Compare prefix
1854+ if (self->prefix > other->prefix)
1855+ return 1;
1856+ else if (self->prefix < other->prefix)
1857+ return -1;
05db64d0 1858+
b952a52b
MT		 1859+ // Both networks are equal
1860+ return 0;
05db64d0
MT		 1861+}
1862+
b952a52b
MT		 1863+LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
1864+ // Either of the start addresses must be in the other subnet
1865+ if (loc_network_match_address(self, &other->first_address))
1866+ return 1;
05db64d0 1867+
b952a52b
MT		 1868+ if (loc_network_match_address(other, &self->first_address))
1869+ return 1;
05db64d0 1870+
b952a52b
MT		 1871+ // Or either of the end addresses is in the other subnet
1872+ if (loc_network_match_address(self, &other->last_address))
1873+ return 1;
05db64d0 1874+
b952a52b
MT		 1875+ if (loc_network_match_address(other, &self->last_address))
1876+ return 1;
05db64d0 1877+
b952a52b 1878+ return 0;
05db64d0
MT		 1879+}
1880+
b952a52b
MT		 1881+LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
1882+ // The prefix must be smaller (this avoids the more complex comparisons later)
1883+ if (self->prefix > other->prefix)
05db64d0
MT		 1884+ return 0;
1885+
b952a52b
MT		 1886 // If the start address of the other network is smaller than this network,
1887 // it cannot be a subnet.
1888- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
1889+ if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
1890 return 0;
1891
1892 // If the end address of the other network is greater than this network,
1893 // it cannot be a subnet.
1894- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
1895+ if (in6_addr_cmp(&self->last_address, &other->last_address) < 0)
1896 return 0;
1897
1898 return 1;
1899 }
1900
1901+LOC_EXPORT int loc_network_subnets(struct loc_network* network,
1902+ struct loc_network** subnet1, struct loc_network** subnet2) {
1903+ int r;
1904+ *subnet1 = NULL;
1905+ *subnet2 = NULL;
05db64d0 1906+
b952a52b
MT		 1907+ // New prefix length
1908+ unsigned int prefix = network->prefix + 1;
1909+
1910+ // Check if the new prefix is valid
1911+ if (valid_prefix(&network->first_address, prefix))
1912+ return -1;
05db64d0 1913+
b952a52b
MT		 1914+ // Create the first half of the network
1915+ r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix);
1916+ if (r)
1917+ return r;
05db64d0 1918+
b952a52b
MT		 1919+ // The next subnet starts after the first one
1920+ struct in6_addr first_address = address_increment(&(*subnet1)->last_address);
05db64d0 1921+
b952a52b
MT		 1922+ // Create the second half of the network
1923+ r = loc_network_new(network->ctx, subnet2, &first_address, prefix);
1924+ if (r)
1925+ return r;
1926+
1927+ // Copy country code
1928+ const char* country_code = loc_network_get_country_code(network);
1929+ if (country_code) {
1930+ loc_network_set_country_code(*subnet1, country_code);
1931+ loc_network_set_country_code(*subnet2, country_code);
1932+ }
1933+
1934+ // Copy ASN
1935+ uint32_t asn = loc_network_get_asn(network);
1936+ if (asn) {
1937+ loc_network_set_asn(*subnet1, asn);
1938+ loc_network_set_asn(*subnet2, asn);
05db64d0
MT		 1939+ }
1940+
b952a52b
MT		 1941+ // Copy flags
1942+ loc_network_set_flag(*subnet1, network->flags);
1943+ loc_network_set_flag(*subnet2, network->flags);
1944+
05db64d0
MT		 1945+ return 0;
1946+}
1947+
b952a52b
MT		 1948+static int __loc_network_exclude(struct loc_network* network,
1949+ struct loc_network* other, struct loc_network_list* list) {
1950+ struct loc_network* subnet1 = NULL;
1951+ struct loc_network* subnet2 = NULL;
05db64d0 1952+
b952a52b 1953+ int r = loc_network_subnets(network, &subnet1, &subnet2);
05db64d0 1954+ if (r)
b952a52b 1955+ goto ERROR;
05db64d0 1956+
b952a52b
MT		 1957+ if (loc_network_cmp(other, subnet1) == 0) {
1958+ r = loc_network_list_push(list, subnet2);
1959+ if (r)
1960+ goto ERROR;
05db64d0 1961+
b952a52b
MT		 1962+ } else if (loc_network_cmp(other, subnet2) == 0) {
1963+ r = loc_network_list_push(list, subnet1);
1964+ if (r)
1965+ goto ERROR;
05db64d0 1966+
b952a52b
MT		 1967+ } else if (loc_network_is_subnet(subnet1, other)) {
1968+ r = loc_network_list_push(list, subnet2);
1969+ if (r)
1970+ goto ERROR;
05db64d0 1971+
b952a52b
MT		 1972+ r = __loc_network_exclude(subnet1, other, list);
1973+ if (r)
1974+ goto ERROR;
05db64d0 1975+
b952a52b
MT		 1976+ } else if (loc_network_is_subnet(subnet2, other)) {
1977+ r = loc_network_list_push(list, subnet1);
1978+ if (r)
1979+ goto ERROR;
05db64d0 1980+
b952a52b
MT		 1981+ r = __loc_network_exclude(subnet2, other, list);
1982+ if (r)
1983+ goto ERROR;
05db64d0 1984+
b952a52b
MT		 1985+ } else {
1986+ ERROR(network->ctx, "We should never get here\n");
1987+ r = 1;
1988+ goto ERROR;
1989+ }
05db64d0 1990+
b952a52b
MT		 1991+ERROR:
1992+ if (subnet1)
1993+ loc_network_unref(subnet1);
05db64d0 1994+
b952a52b
MT		 1995+ if (subnet2)
1996+ loc_network_unref(subnet2);
05db64d0 1997+
b952a52b
MT		 1998+ return r;
1999+}
05db64d0 2000+
b952a52b
MT		 2001+static int __loc_network_exclude_to_list(struct loc_network* self,
2002+ struct loc_network* other, struct loc_network_list* list) {
2003+ // Other must be a subnet of self
2004+ if (!loc_network_is_subnet(self, other)) {
2005+ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
2006+
2007+ // Exit silently
2008+ return 0;
2009+ }
2010+
2011+ // We cannot perform this operation if both networks equal
2012+ if (loc_network_cmp(self, other) == 0) {
2013+ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
2014+
2015+ // Exit silently
2016+ return 0;
2017+ }
05db64d0 2018+
b952a52b
MT		 2019+ return __loc_network_exclude(self, other, list);
2020+}
05db64d0 2021+
b952a52b
MT		 2022+LOC_EXPORT struct loc_network_list* loc_network_exclude(
2023+ struct loc_network* self, struct loc_network* other) {
2024+ struct loc_network_list* list;
05db64d0 2025+
b952a52b
MT		 2026+#ifdef ENABLE_DEBUG
2027+ char* n1 = loc_network_str(self);
2028+ char* n2 = loc_network_str(other);
05db64d0 2029+
b952a52b 2030+ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
05db64d0 2031+
b952a52b
MT		 2032+ free(n1);
2033+ free(n2);
2034+#endif
05db64d0 2035+
b952a52b
MT		 2036+ // Create a new list with the result
2037+ int r = loc_network_list_new(self->ctx, &list);
2038+ if (r) {
2039+ ERROR(self->ctx, "Could not create network list: %d\n", r);
05db64d0 2040+
b952a52b
MT		 2041+ return NULL;
2042+ }
05db64d0 2043+
b952a52b
MT		 2044+ r = __loc_network_exclude_to_list(self, other, list);
2045+ if (r) {
2046+ loc_network_list_unref(list);
05db64d0 2047+
b952a52b
MT		 2048+ return NULL;
2049+ }
05db64d0 2050+
b952a52b
MT		 2051+ // Return the result
2052+ return list;
2053+}
05db64d0 2054+
b952a52b
MT		 2055+LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
2056+ struct loc_network* network, struct loc_network_list* list) {
2057+ struct loc_network_list* to_check;
05db64d0 2058+
b952a52b
MT		 2059+ // Create a new list with all networks to look at
2060+ int r = loc_network_list_new(network->ctx, &to_check);
2061+ if (r)
2062+ return NULL;
05db64d0 2063+
b952a52b
MT		 2064+ struct loc_network* subnet = NULL;
2065+ struct loc_network_list* subnets = NULL;
05db64d0 2066+
b952a52b
MT		 2067+ for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
2068+ subnet = loc_network_list_get(list, i);
2069+
2070+ // Find all excluded networks
2071+ if (!loc_network_list_contains(to_check, subnet)) {
2072+ r = __loc_network_exclude_to_list(network, subnet, to_check);
05db64d0 2073+ if (r) {
b952a52b
MT		 2074+ loc_network_list_unref(to_check);
2075+ loc_network_unref(subnet);
05db64d0 2076+
05db64d0
MT		 2077+ return NULL;
2078+ }
05db64d0
MT		 2079+ }
2080+
b952a52b
MT		 2081+ // Cleanup
2082+ loc_network_unref(subnet);
05db64d0 2083+ }
05db64d0 2084+
b952a52b
MT		 2085+ r = loc_network_list_new(network->ctx, &subnets);
2086+ if (r) {
2087+ loc_network_list_unref(to_check);
2088+ return NULL;
05db64d0 2089+ }
05db64d0 2090+
b952a52b 2091+ off_t smallest_subnet = 0;
05db64d0 2092+
b952a52b
MT		 2093+ while (!loc_network_list_empty(to_check)) {
2094+ struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check);
05db64d0 2095+
b952a52b
MT		 2096+ // Check whether the subnet to check is part of the input list
2097+ if (loc_network_list_contains(list, subnet_to_check)) {
2098+ loc_network_unref(subnet_to_check);
2099+ continue;
2100+ }
05db64d0 2101+
b952a52b
MT		 2102+ // Marks whether this subnet passed all checks
2103+ int passed = 1;
05db64d0 2104+
b952a52b
MT		 2105+ for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) {
2106+ subnet = loc_network_list_get(list, i);
2107+
2108+ // Drop this subnet if is a subnet of another subnet
2109+ if (loc_network_is_subnet(subnet, subnet_to_check)) {
2110+ passed = 0;
2111+ loc_network_unref(subnet);
2112+ break;
2113+ }
05db64d0 2114+
b952a52b
MT		 2115+ // Break it down if it overlaps
2116+ if (loc_network_overlaps(subnet, subnet_to_check)) {
2117+ passed = 0;
05db64d0 2118+
b952a52b 2119+ __loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
05db64d0 2120+
b952a52b
MT		 2121+ loc_network_unref(subnet);
2122+ break;
2123+ }
05db64d0 2124+
b952a52b
MT		 2125+ // If the subnet is strictly greater, we do not need to continue the search
2126+ r = loc_network_cmp(subnet, subnet_to_check);
2127+ if (r > 0) {
2128+ loc_network_unref(subnet);
2129+ break;
05db64d0 2130+
b952a52b
MT		 2131+ // If it is strictly smaller, we can continue the search from here next
2132+ // time because all networks that are to be checked can only be larger
2133+ // than this one.
2134+ } else if (r < 0) {
2135+ smallest_subnet = i;
2136+ }
05db64d0 2137+
b952a52b
MT		 2138+ loc_network_unref(subnet);
2139+ }
05db64d0 2140+
b952a52b
MT		 2141+ if (passed) {
2142+ r = loc_network_list_push(subnets, subnet_to_check);
2143+ }
05db64d0 2144+
b952a52b
MT		 2145+ loc_network_unref(subnet_to_check);
2146+ }
05db64d0 2147+
b952a52b
MT		 2148+ loc_network_list_unref(to_check);
2149+
2150+ return subnets;
05db64d0
MT		 2151+}
2152+
b952a52b
MT		 2153 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
2154 // Add country code
2155 loc_country_code_copy(dbobj->country_code, network->country_code);
2156@@ -474,7 +813,7 @@ struct loc_network_tree_node {
2157 struct loc_network* network;
2158 };
05db64d0 2159
b952a52b
MT		 2160-LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
2161+int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
2162 struct loc_network_tree* t = calloc(1, sizeof(*t));
2163 if (!t)
2164 return -ENOMEM;
2165@@ -494,7 +833,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
05db64d0
MT		 2166 return 0;
2167 }
05db64d0 2168
b952a52b
MT		 2169-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
2170+struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
2171 return loc_network_tree_node_ref(tree->root);
05db64d0
MT		 2172 }
2173
b952a52b
MT		 2174@@ -566,7 +905,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
2175 return 0;
05db64d0
MT		 2176 }
2177
b952a52b
MT		 2178-LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
2179+int loc_network_tree_walk(struct loc_network_tree* tree,
2180 int(*filter_callback)(struct loc_network* network, void* data),
2181 int(*callback)(struct loc_network* network, void* data), void* data) {
2182 return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
2183@@ -581,7 +920,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
2184 free(tree);
05db64d0
MT		 2185 }
2186
b952a52b
MT		 2187-LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
2188+struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
2189 if (--tree->refcount > 0)
2190 return tree;
05db64d0 2191
b952a52b
MT		 2192@@ -602,13 +941,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
2193 return 0;
05db64d0
MT		 2194 }
2195
b952a52b
MT		 2196-LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
2197+int loc_network_tree_dump(struct loc_network_tree* tree) {
2198 DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
05db64d0 2199
b952a52b
MT		 2200 return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
2201 }
2202
2203-LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
2204+int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
2205 DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
05db64d0 2206
b952a52b
MT		 2207 struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
2208@@ -639,7 +978,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
05db64d0 2209 return 0;
b952a52b 2210 }
05db64d0 2211
b952a52b
MT		 2212-LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
2213+size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
2214 size_t counter = 0;
78a6918d 2215
b952a52b
MT		 2216 int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
2217@@ -661,11 +1000,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
2218 return counter;
2219 }
78a6918d 2220
b952a52b
MT		 2221-LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
2222+size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
2223 return __loc_network_tree_count_nodes(tree->root);
2224 }
78a6918d 2225
b952a52b
MT		 2226-LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
2227+int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
2228 struct loc_network_tree_node* n = calloc(1, sizeof(*n));
2229 if (!n)
2230 return -ENOMEM;
2231@@ -680,7 +1019,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
2232 return 0;
2233 }
78a6918d 2234
b952a52b
MT		 2235-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
2236+struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
2237 if (node)
2238 node->refcount++;
78a6918d 2239
b952a52b
MT		 2240@@ -703,7 +1042,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
2241 free(node);
2242 }
78a6918d 2243
b952a52b
MT		 2244-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
2245+struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
2246 if (!node)
2247 return NULL;
78a6918d 2248
b952a52b
MT		 2249@@ -714,7 +1053,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
2250 return NULL;
2251 }
78a6918d 2252
b952a52b
MT		 2253-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
2254+struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
2255 if (index == 0)
2256 node = node->zero;
2257 else
2258@@ -726,10 +1065,10 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
2259 return loc_network_tree_node_ref(node);
2260 }
78a6918d 2261
b952a52b
MT		 2262-LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
2263+int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
2264 return (!!node->network);
2265 }
78a6918d 2266
b952a52b
MT		 2267-LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2268+struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2269 return loc_network_ref(node->network);
2270 }
2271diff --git a/src/perl/Location.xs b/src/perl/Location.xs
2272index dcf3f0d..b7676d2 100644
2273--- a/src/perl/Location.xs
2274+++ b/src/perl/Location.xs
2275@@ -125,7 +125,7 @@ database_countries(db)
2276 PPCODE:
2277 // Create Database enumerator
2278 struct loc_database_enumerator* enumerator;
2279- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
2280+ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
78a6918d 2281
b952a52b
MT		 2282 if (err) {
2283 croak("Could not create a database enumerator\n");
2284diff --git a/src/python/database.c b/src/python/database.c
4a388bc9 2285index 1013a58..0aa03cc 100644
b952a52b
MT		 2286--- a/src/python/database.c
2287+++ b/src/python/database.c
2288@@ -17,6 +17,8 @@
2289 #include <Python.h>
78a6918d 2290
b952a52b
MT		 2291 #include <loc/libloc.h>
2292+#include <loc/as.h>
2293+#include <loc/as-list.h>
2294 #include <loc/database.h>
78a6918d 2295
b952a52b
MT		 2296 #include "locationmodule.h"
2297@@ -207,10 +209,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
2298 return (PyObject*)self;
2299 }
78a6918d 2300
b952a52b
MT		 2301-static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
2302+static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
2303 struct loc_database_enumerator* enumerator;
78a6918d 2304
b952a52b
MT		 2305- int r = loc_database_enumerator_new(&enumerator, self->db, what);
2306+ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
2307 if (r) {
2308 PyErr_SetFromErrno(PyExc_SystemError);
2309 return NULL;
2310@@ -223,7 +225,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
2311 }
78a6918d 2312
b952a52b
MT		 2313 static PyObject* Database_ases(DatabaseObject* self) {
2314- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
2315+ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
2316 }
78a6918d 2317
b952a52b
MT		 2318 static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2319@@ -234,7 +236,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2320
2321 struct loc_database_enumerator* enumerator;
2322
2323- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
2324+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
2325 if (r) {
2326 PyErr_SetFromErrno(PyExc_SystemError);
2327 return NULL;
2328@@ -250,44 +252,142 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
78a6918d
MT		 2329 }
2330
b952a52b
MT		 2331 static PyObject* Database_networks(DatabaseObject* self) {
2332- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
2333+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
78a6918d
MT		 2334+}
2335+
b952a52b
MT		 2336+static PyObject* Database_networks_flattened(DatabaseObject *self) {
2337+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
2338 }
2339
2340 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
2341- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
2342- const char* country_code = NULL;
2343- unsigned int asn = 0;
2344+ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
2345+ PyObject* country_codes = NULL;
2346+ PyObject* asn_list = NULL;
2347 int flags = 0;
2348 int family = 0;
2349+ int flatten = 0;
2350
2351- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
2352+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
2353+ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
2354 return NULL;
2355
2356 struct loc_database_enumerator* enumerator;
2357- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
2358+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
2359+ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
2360 if (r) {
2361 PyErr_SetFromErrno(PyExc_SystemError);
2362 return NULL;
2363 }
2364
2365 // Set country code we are searching for
2366- if (country_code) {
2367- r = loc_database_enumerator_set_country_code(enumerator, country_code);
2368+ if (country_codes) {
2369+ struct loc_country_list* countries;
2370+ r = loc_country_list_new(loc_ctx, &countries);
2371+ if (r) {
2372+ PyErr_SetString(PyExc_SystemError, "Could not create country list");
2373+ return NULL;
2374+ }
2375+
4a388bc9 2376+ for (int i = 0; i < PyList_Size(country_codes); i++) {
b952a52b
MT		 2377+ PyObject* item = PyList_GetItem(country_codes, i);
2378+
2379+ if (!PyUnicode_Check(item)) {
2380+ PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
2381+ loc_country_list_unref(countries);
2382+ return NULL;
2383+ }
2384+
2385+ const char* country_code = PyUnicode_AsUTF8(item);
2386+
2387+ struct loc_country* country;
2388+ r = loc_country_new(loc_ctx, &country, country_code);
2389+ if (r) {
2390+ if (r == -EINVAL) {
2391+ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
2392+ } else {
2393+ PyErr_SetString(PyExc_SystemError, "Could not create country");
2394+ }
2395+
2396+ loc_country_list_unref(countries);
2397+ return NULL;
2398+ }
2399+
2400+ // Append it to the list
2401+ r = loc_country_list_append(countries, country);
2402+ if (r) {
2403+ PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
78a6918d 2404+
b952a52b
MT		 2405+ loc_country_list_unref(countries);
2406+ loc_country_unref(country);
2407+ return NULL;
2408+ }
2409+
2410+ loc_country_unref(country);
2411+ }
2412
2413+ r = loc_database_enumerator_set_countries(enumerator, countries);
78a6918d
MT		 2414 if (r) {
2415 PyErr_SetFromErrno(PyExc_SystemError);
b952a52b 2416+
78a6918d
MT		 2417+ loc_country_list_unref(countries);
2418 return NULL;
2419 }
b952a52b
MT		 2420+
2421+ loc_country_list_unref(countries);
2422 }
78a6918d 2423
b952a52b
MT		 2424 // Set the ASN we are searching for
2425- if (asn) {
2426- r = loc_database_enumerator_set_asn(enumerator, asn);
2427+ if (asn_list) {
2428+ struct loc_as_list* asns;
2429+ r = loc_as_list_new(loc_ctx, &asns);
2430+ if (r) {
2431+ PyErr_SetString(PyExc_SystemError, "Could not create AS list");
2432+ return NULL;
2433+ }
2434+
4a388bc9 2435+ for (int i = 0; i < PyList_Size(asn_list); i++) {
b952a52b
MT		 2436+ PyObject* item = PyList_GetItem(asn_list, i);
2437+
2438+ if (!PyLong_Check(item)) {
2439+ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
78a6918d 2440
b952a52b
MT		 2441+ loc_as_list_unref(asns);
2442+ return NULL;
2443+ }
78a6918d 2444+
b952a52b 2445+ unsigned long number = PyLong_AsLong(item);
78a6918d 2446+
b952a52b
MT		 2447+ struct loc_as* as;
2448+ r = loc_as_new(loc_ctx, &as, number);
2449+ if (r) {
2450+ PyErr_SetString(PyExc_SystemError, "Could not create AS");
78a6918d 2451+
b952a52b
MT		 2452+ loc_as_list_unref(asns);
2453+ loc_as_unref(as);
2454+ return NULL;
2455+ }
78a6918d 2456+
b952a52b
MT		 2457+ r = loc_as_list_append(asns, as);
2458+ if (r) {
2459+ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
78a6918d 2460+
b952a52b
MT		 2461+ loc_as_list_unref(asns);
2462+ loc_as_unref(as);
2463+ return NULL;
2464+ }
78a6918d 2465+
b952a52b
MT		 2466+ loc_as_unref(as);
2467+ }
2468+
2469+ r = loc_database_enumerator_set_asns(enumerator, asns);
2470 if (r) {
2471 PyErr_SetFromErrno(PyExc_SystemError);
2472+
2473+ loc_as_list_unref(asns);
2474 return NULL;
2475 }
2476+
2477+ loc_as_list_unref(asns);
2478 }
78a6918d 2479
b952a52b
MT		 2480 // Set the flags we are searching for
2481@@ -317,7 +417,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
78a6918d
MT		 2482 }
2483
b952a52b
MT		 2484 static PyObject* Database_countries(DatabaseObject* self) {
2485- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
2486+ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
2487 }
2488
2489 static struct PyMethodDef Database_methods[] = {
2490@@ -403,6 +503,13 @@ static struct PyGetSetDef Database_getsetters[] = {
78a6918d
MT		 2491 NULL,
2492 NULL,
2493 },
2494+ {
b952a52b
MT		 2495+ "networks_flattened",
2496+ (getter)Database_networks_flattened,
78a6918d
MT		 2497+ NULL,
2498+ NULL,
2499+ NULL,
2500+ },
2501 {
b952a52b
MT		 2502 "vendor",
2503 (getter)Database_get_vendor,
2504diff --git a/src/python/downloader.py b/src/python/downloader.py
2505index 87bbb68..05f7872 100644
2506--- a/src/python/downloader.py
2507+++ b/src/python/downloader.py
2508@@ -119,8 +119,8 @@ class Downloader(object):
2509
2510 headers = {}
2511 if timestamp:
2512- headers["If-Modified-Since"] = timestamp.strftime(
2513- "%a, %d %b %Y %H:%M:%S GMT",
2514+ headers["If-Modified-Since"] = time.strftime(
2515+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
2516 )
2517
2518 t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
2519@@ -195,7 +195,7 @@ class Downloader(object):
2520 db = Database(f.name)
2521
2522 # Database is not recent
2523- if timestamp and db.created_at < timestamp.timestamp():
2524+ if timestamp and db.created_at < timestamp:
2525 return False
78a6918d 2526
b952a52b 2527 log.info("Downloaded new database from %s" % (time.strftime(
78a6918d 2528diff --git a/src/python/export.py b/src/python/export.py
b952a52b 2529index d15c6f0..f0eae26 100644
78a6918d
MT		 2530--- a/src/python/export.py
2531+++ b/src/python/export.py
b952a52b
MT		 2532@@ -29,7 +29,7 @@ import _location
2533 log = logging.getLogger("location.export")
2534 log.propagate = 1
2535
2536-flags = {
2537+FLAGS = {
2538 _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
2539 _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
2540 _location.NETWORK_FLAG_ANYCAST : "A3",
78a6918d
MT		 2541@@ -39,11 +39,8 @@ class OutputWriter(object):
2542 suffix = "networks"
2543 mode = "w"
2544
2545- def __init__(self, f, prefix=None, flatten=True):
2546- self.f, self.prefix, self.flatten = f, prefix, flatten
2547-
2548- # The previously written network
2549- self._last_network = None
2550+ def __init__(self, f, prefix=None):
2551+ self.f, self.prefix = f, prefix
2552
2553 # Immediately write the header
2554 self._write_header()
2555@@ -60,18 +57,6 @@ class OutputWriter(object):
2556 def __repr__(self):
2557 return "<%s f=%s>" % (self.__class__.__name__, self.f)
2558
2559- def _flatten(self, network):
2560- """
2561- Checks if the given network needs to be written to file,
2562- or if it is a subnet of the previously written network.
2563- """
2564- if self._last_network and network.is_subnet_of(self._last_network):
2565- return True
2566-
2567- # Remember this network for the next call
2568- self._last_network = network
2569- return False
2570-
2571 def _write_header(self):
2572 """
2573 The header of the file
b952a52b 2574@@ -84,16 +69,8 @@ class OutputWriter(object):
78a6918d
MT		 2575 """
2576 pass
2577
2578- def _write_network(self, network):
2579- self.f.write("%s\n" % network)
2580-
2581 def write(self, network):
2582- if self.flatten and self._flatten(network):
b952a52b 2583- log.debug("Skipping writing network %s" % network)
78a6918d
MT		 2584- return
2585-
b952a52b
MT		 2586- # Write the network to file
2587- self._write_network(network)
78a6918d
MT		 2588+ self.f.write("%s\n" % network)
2589
2590 def finish(self):
2591 """
b952a52b 2592@@ -114,7 +91,7 @@ class IpsetOutputWriter(OutputWriter):
78a6918d
MT		 2593 def _write_header(self):
2594 self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
2595
2596- def _write_network(self, network):
2597+ def write(self, network):
2598 self.f.write("add %s %s\n" % (self.prefix, network))
2599
2600
b952a52b 2601@@ -130,7 +107,7 @@ class NftablesOutputWriter(OutputWriter):
78a6918d
MT		 2602 def _write_footer(self):
2603 self.f.write("}\n")
2604
2605- def _write_network(self, network):
2606+ def write(self, network):
b952a52b 2607 self.f.write(" %s,\n" % network)
78a6918d 2608
78a6918d 2609
b952a52b
MT		 2610@@ -142,14 +119,9 @@ class XTGeoIPOutputWriter(OutputWriter):
2611 suffix = "iv"
2612 mode = "wb"
78a6918d 2613
b952a52b
MT		 2614- def _write_network(self, network):
2615- for address in (network.first_address, network.last_address):
2616- # Convert this into a string of bits
2617- bytes = socket.inet_pton(
2618- network.family, address,
2619- )
2620-
2621- self.f.write(bytes)
2622+ def write(self, network):
2623+ self.f.write(network._first_address)
2624+ self.f.write(network._last_address)
78a6918d 2625
78a6918d 2626
b952a52b
MT		 2627 formats = {
2628@@ -185,8 +157,14 @@ class Exporter(object):
78a6918d 2629
b952a52b 2630 writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
78a6918d 2631
b952a52b
MT		 2632+ # Filter countries from special country codes
2633+ country_codes = [
2634+ country_code for country_code in countries if not country_code in FLAGS.values()
2635+ ]
2636+
2637 # Get all networks that match the family
2638- networks = self.db.search_networks(family=family)
2639+ networks = self.db.search_networks(family=family,
2640+ country_codes=country_codes, asns=asns, flatten=True)
78a6918d 2641
b952a52b
MT		 2642 # Walk through all networks
2643 for network in networks:
2644@@ -203,10 +181,10 @@ class Exporter(object):
2645 pass
78a6918d 2646
b952a52b
MT		 2647 # Handle flags
2648- for flag in flags:
2649+ for flag in FLAGS:
2650 if network.has_flag(flag):
2651 # Fetch the "fake" country code
2652- country = flags[flag]
2653+ country = FLAGS[flag]
0be475ca 2654
b952a52b
MT		 2655 try:
2656 writers[country].write(network)
2657diff --git a/src/python/importer.py b/src/python/importer.py
2658index f19db4b..5f46bc3 100644
2659--- a/src/python/importer.py
2660+++ b/src/python/importer.py
2661@@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
2662 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
0be475ca 2663
b952a52b
MT		 2664 # Latin America and Caribbean Network Information Centre
2665- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2666+ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
0be475ca 2667
b952a52b
MT		 2668 # Réseaux IP Européens
2669 #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
2670diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2671index 1467923..2dec89e 100644
2672--- a/src/python/location-importer.in
2673+++ b/src/python/location-importer.in
2674@@ -152,6 +152,7 @@ class CLI(object):
2675 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
2676 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
2677 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
2678+ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
2679
2680 -- autnums
2681 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
2682@@ -165,6 +166,7 @@ class CLI(object):
2683 -- networks
2684 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
2685 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
2686+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
2687 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
0be475ca 2688
b952a52b
MT		 2689 -- overrides
2690@@ -188,6 +190,8 @@ class CLI(object):
2691 );
2692 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
2693 ON network_overrides(network);
2694+ CREATE INDEX IF NOT EXISTS network_overrides_search
2695+ ON network_overrides USING GIST(network inet_ops);
2696 """)
0be475ca 2697
b952a52b
MT		 2698 return db
2699@@ -234,32 +238,24 @@ class CLI(object):
0be475ca 2700
b952a52b
MT		 2701 # Select all known networks
2702 rows = self.db.query("""
2703- -- Get a (sorted) list of all known networks
2704- WITH known_networks AS (
2705- SELECT network FROM announcements
2706- UNION
2707- SELECT network FROM networks
2708- ORDER BY network
2709- )
0be475ca 2710-
b952a52b
MT		 2711 -- Return a list of those networks enriched with all
2712 -- other information that we store in the database
2713 SELECT
2714- DISTINCT ON (known_networks.network)
2715- known_networks.network AS network,
2716- announcements.autnum AS autnum,
2717+ DISTINCT ON (network)
2718+ network,
2719+ autnum,
0be475ca 2720
b952a52b
MT		 2721 -- Country
2722 COALESCE(
2723 (
2724 SELECT country FROM network_overrides overrides
2725- WHERE announcements.network <<= overrides.network
2726+ WHERE networks.network <<= overrides.network
2727 ORDER BY masklen(overrides.network) DESC
2728 LIMIT 1
2729 ),
2730 (
2731 SELECT country FROM autnum_overrides overrides
2732- WHERE announcements.autnum = overrides.number
2733+ WHERE networks.autnum = overrides.number
2734 ),
2735 networks.country
2736 ) AS country,
2737@@ -268,50 +264,67 @@ class CLI(object):
2738 COALESCE(
2739 (
2740 SELECT is_anonymous_proxy FROM network_overrides overrides
2741- WHERE announcements.network <<= overrides.network
2742+ WHERE networks.network <<= overrides.network
2743 ORDER BY masklen(overrides.network) DESC
2744 LIMIT 1
2745 ),
2746 (
2747 SELECT is_anonymous_proxy FROM autnum_overrides overrides
2748- WHERE announcements.autnum = overrides.number
2749+ WHERE networks.autnum = overrides.number
2750 ),
2751 FALSE
2752 ) AS is_anonymous_proxy,
2753 COALESCE(
2754 (
2755 SELECT is_satellite_provider FROM network_overrides overrides
2756- WHERE announcements.network <<= overrides.network
2757+ WHERE networks.network <<= overrides.network
2758 ORDER BY masklen(overrides.network) DESC
2759 LIMIT 1
2760 ),
2761 (
2762 SELECT is_satellite_provider FROM autnum_overrides overrides
2763- WHERE announcements.autnum = overrides.number
2764+ WHERE networks.autnum = overrides.number
2765 ),
2766 FALSE
2767 ) AS is_satellite_provider,
2768 COALESCE(
2769 (
2770 SELECT is_anycast FROM network_overrides overrides
2771- WHERE announcements.network <<= overrides.network
2772+ WHERE networks.network <<= overrides.network
2773 ORDER BY masklen(overrides.network) DESC
2774 LIMIT 1
2775 ),
2776 (
2777 SELECT is_anycast FROM autnum_overrides overrides
2778- WHERE announcements.autnum = overrides.number
2779+ WHERE networks.autnum = overrides.number
2780 ),
2781 FALSE
2782- ) AS is_anycast,
0be475ca 2783-
b952a52b
MT		 2784- -- Must be part of returned values for ORDER BY clause
2785- masklen(announcements.network) AS sort_a,
2786- masklen(networks.network) AS sort_b
2787- FROM known_networks
2788- LEFT JOIN announcements ON known_networks.network <<= announcements.network
2789- LEFT JOIN networks ON known_networks.network <<= networks.network
2790- ORDER BY known_networks.network, sort_a DESC, sort_b DESC
2791+ ) AS is_anycast
2792+ FROM (
2793+ SELECT
2794+ known_networks.network AS network,
2795+ announcements.autnum AS autnum,
2796+ networks.country AS country,
0be475ca 2797+
b952a52b
MT		 2798+ -- Must be part of returned values for ORDER BY clause
2799+ masklen(announcements.network) AS sort_a,
2800+ masklen(networks.network) AS sort_b
2801+ FROM (
2802+ SELECT network FROM announcements
2803+ UNION ALL
2804+ SELECT network FROM networks
2805+ UNION ALL
2806+ SELECT network FROM network_overrides
2807+ ) known_networks
2808+ LEFT JOIN
2809+ announcements ON known_networks.network <<= announcements.network
2810+ LEFT JOIN
2811+ networks ON known_networks.network <<= networks.network
2812+ ORDER BY
2813+ known_networks.network,
2814+ sort_a DESC,
2815+ sort_b DESC
2816+ ) networks
2817 """)
0be475ca 2818
b952a52b
MT		 2819 for row in rows:
2820@@ -363,6 +376,16 @@ class CLI(object):
2821 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
2822 ON COMMIT DROP;
2823 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
0be475ca 2824+
b952a52b
MT		 2825+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
2826+ ON COMMIT DROP;
2827+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
2828+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
2829+ """)
2830+
2831+ # Remove all previously imported content
2832+ self.db.execute("""
2833+ TRUNCATE TABLE networks;
2834 """)
0be475ca 2835
b952a52b
MT		 2836 for source in location.importer.WHOIS_SOURCES:
2837@@ -370,31 +393,72 @@ class CLI(object):
2838 for block in f:
2839 self._parse_block(block)
0be475ca 2840
b952a52b
MT		 2841+ # Process all parsed networks from every RIR we happen to have access to,
2842+ # insert the largest network chunks into the networks table immediately...
2843+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
0be475ca 2844+
b952a52b
MT		 2845+ for family in (row.family for row in families):
2846+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
0be475ca 2847+
b952a52b
MT		 2848+ self.db.execute("INSERT INTO networks(network, country) \
2849+ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
0be475ca 2850+
b952a52b
MT		 2851+ # ... determine any other prefixes for this network family, ...
2852+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
2853+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
0be475ca 2854+
b952a52b
MT		 2855+ # ... and insert networks with this prefix in case they provide additional
2856+ # information (i. e. subnet of a larger chunk with a different country)
2857+ for prefix in (row.prefix for row in prefixes):
2858+ self.db.execute("""
2859+ WITH candidates AS (
2860+ SELECT
2861+ _rirdata.network,
2862+ _rirdata.country
2863+ FROM
2864+ _rirdata
2865+ WHERE
2866+ family(_rirdata.network) = %s
2867+ AND
2868+ masklen(_rirdata.network) = %s
2869+ ),
2870+ filtered AS (
2871+ SELECT
2872+ DISTINCT ON (c.network)
2873+ c.network,
2874+ c.country,
2875+ masklen(networks.network),
2876+ networks.country AS parent_country
2877+ FROM
2878+ candidates c
2879+ LEFT JOIN
2880+ networks
2881+ ON
2882+ c.network << networks.network
2883+ ORDER BY
2884+ c.network,
2885+ masklen(networks.network) DESC NULLS LAST
2886+ )
2887+ INSERT INTO
2888+ networks(network, country)
2889+ SELECT
2890+ network,
2891+ country
2892+ FROM
2893+ filtered
2894+ WHERE
2895+ parent_country IS NULL
2896+ OR
2897+ country <> parent_country
2898+ ON CONFLICT DO NOTHING""",
2899+ family, prefix,
2900+ )
0be475ca 2901+
b952a52b
MT		 2902 self.db.execute("""
2903 INSERT INTO autnums(number, name)
2904 SELECT _autnums.number, _organizations.name FROM _autnums
2905 JOIN _organizations ON _autnums.organization = _organizations.handle
2906- ON CONFLICT (number) DO UPDATE SET name = excluded.name
2907- """)
2908-
2909- self.db.execute("""
2910- --- Purge any redundant entries
2911- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
2912- AS
2913- SELECT network FROM networks candidates
2914- WHERE EXISTS (
2915- SELECT FROM networks
2916- WHERE
2917- networks.network << candidates.network
2918- AND
2919- networks.country = candidates.country
2920- );
0be475ca 2921-
b952a52b 2922- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
0be475ca 2923-
b952a52b
MT		 2924- DELETE FROM networks WHERE EXISTS (
2925- SELECT FROM _garbage WHERE networks.network = _garbage.network
2926- );
2927+ ON CONFLICT (number) DO UPDATE SET name = excluded.name;
2928 """)
0be475ca 2929
b952a52b
MT		 2930 # Download all extended sources
2931@@ -405,6 +469,69 @@ class CLI(object):
2932 for line in f:
2933 self._parse_line(line)
0be475ca 2934
b952a52b
MT		 2935+ def _check_parsed_network(self, network):
2936+ """
2937+ Assistive function to detect and subsequently sort out parsed
2938+ networks from RIR data (both Whois and so-called "extended sources"),
2939+ which are or have...
0be475ca 2940+
b952a52b
MT		 2941+ (a) not globally routable (RFC 1918 space, et al.)
2942+ (b) covering a too large chunk of the IP address space (prefix length
2943+ is < 7 for IPv4 networks, and < 10 for IPv6)
2944+ (c) "0.0.0.0" or "::" as a network address
2945+ (d) are too small for being publicly announced (we have decided not to
2946+ process them at the moment, as they significantly enlarge our
2947+ database without providing very helpful additional information)
0be475ca 2948+
b952a52b
MT		 2949+ This unfortunately is necessary due to brain-dead clutter across
2950+ various RIR databases, causing mismatches and eventually disruptions.
0be475ca 2951+
b952a52b
MT		 2952+ We will return False in case a network is not suitable for adding
2953+ it to our database, and True otherwise.
2954+ """
0be475ca 2955+
b952a52b
MT		 2956+ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
2957+ return False
0be475ca 2958+
b952a52b
MT		 2959+ if not network.is_global:
2960+ log.warning("Skipping non-globally routable network: %s" % network)
2961+ return False
0be475ca 2962+
b952a52b
MT		 2963+ if network.version == 4:
2964+ if network.prefixlen < 7:
2965+ log.warning("Skipping too big IP chunk: %s" % network)
2966+ return False
0be475ca 2967+
b952a52b
MT		 2968+ if network.prefixlen > 24:
2969+ log.debug("Skipping network too small to be publicly announced: %s" % network)
2970+ return False
0be475ca 2971+
b952a52b
MT		 2972+ if str(network.network_address) == "0.0.0.0":
2973+ log.warning("Skipping network based on 0.0.0.0: %s" % network)
2974+ return False
0be475ca 2975+
b952a52b
MT		 2976+ elif network.version == 6:
2977+ if network.prefixlen < 10:
2978+ log.warning("Skipping too big IP chunk: %s" % network)
2979+ return False
0be475ca 2980+
b952a52b
MT		 2981+ if network.prefixlen > 48:
2982+ log.debug("Skipping network too small to be publicly announced: %s" % network)
2983+ return False
0be475ca 2984+
b952a52b
MT		 2985+ if str(network.network_address) == "::":
2986+ log.warning("Skipping network based on '::': %s" % network)
2987+ return False
2988+
2989+ else:
2990+ # This should not happen...
2991+ log.warning("Skipping network of unknown family, this should not happen: %s" % network)
2992+ return False
2993+
2994+ # In case we have made it here, the network is considered to
2995+ # be suitable for libloc consumption...
2996+ return True
2997+
2998 def _parse_block(self, block):
2999 # Get first line to find out what type of block this is
3000 line = block[0]
3001@@ -433,7 +560,7 @@ class CLI(object):
3002 autnum["asn"] = m.group(2)
0be475ca 3003
b952a52b
MT		 3004 elif key == "org":
3005- autnum[key] = val
3006+ autnum[key] = val.upper()
0be475ca 3007
b952a52b
MT		 3008 # Skip empty objects
3009 if not autnum:
3010@@ -447,15 +574,22 @@ class CLI(object):
3011 )
0be475ca 3012
b952a52b
MT		 3013 def _parse_inetnum_block(self, block):
3014- logging.debug("Parsing inetnum block:")
3015+ log.debug("Parsing inetnum block:")
0be475ca 3016
b952a52b
MT		 3017 inetnum = {}
3018 for line in block:
3019- logging.debug(line)
3020+ log.debug(line)
0be475ca 3021
b952a52b
MT		 3022 # Split line
3023 key, val = split_line(line)
0be475ca 3024
b952a52b
MT		 3025+ # Filter any inetnum records which are only referring to IP space
3026+ # not managed by that specific RIR...
3027+ if key == "netname":
3028+ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
3029+ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
3030+ return
0be475ca 3031+
b952a52b
MT		 3032 if key == "inetnum":
3033 start_address, delim, end_address = val.partition("-")
3034
3035@@ -467,7 +601,7 @@ class CLI(object):
3036 start_address = ipaddress.ip_address(start_address)
3037 end_address = ipaddress.ip_address(end_address)
3038 except ValueError:
3039- logging.warning("Could not parse line: %s" % line)
3040+ log.warning("Could not parse line: %s" % line)
3041 return
3042
3043 # Set prefix to default
3044@@ -484,23 +618,24 @@ class CLI(object):
3045 inetnum[key] = val
0be475ca 3046
b952a52b
MT		 3047 elif key == "country":
3048- if val == "UNITED STATES":
3049- val = "US"
0be475ca 3050-
b952a52b 3051 inetnum[key] = val.upper()
0be475ca 3052
b952a52b
MT		 3053 # Skip empty objects
3054- if not inetnum:
3055+ if not inetnum or not "country" in inetnum:
3056+ return
3057+
3058+ # Skip objects with bogus country code 'ZZ'
3059+ if inetnum.get("country") == "ZZ":
3060+ log.warning("Skipping network with bogus country 'ZZ': %s" % \
3061+ (inetnum.get("inet6num") or inetnum.get("inetnum")))
3062 return
0be475ca 3063
b952a52b 3064 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
0be475ca 3065
b952a52b
MT		 3066- # Bail out in case we have processed a non-public IP network
3067- if network.is_private:
3068- logging.warning("Skipping non-globally routable network: %s" % network)
3069+ if not self._check_parsed_network(network):
3070 return
0be475ca 3071
b952a52b
MT		 3072- self.db.execute("INSERT INTO networks(network, country) \
3073+ self.db.execute("INSERT INTO _rirdata(network, country) \
3074 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
3075 "%s" % network, inetnum.get("country"),
3076 )
3077@@ -511,7 +646,9 @@ class CLI(object):
3078 # Split line
3079 key, val = split_line(line)
0be475ca 3080
b952a52b
MT		 3081- if key in ("organisation", "org-name"):
3082+ if key == "organisation":
3083+ org[key] = val.upper()
3084+ elif key == "org-name":
3085 org[key] = val
0be475ca 3086
b952a52b
MT		 3087 # Skip empty objects
3088@@ -581,6 +718,9 @@ class CLI(object):
3089 log.warning("Invalid IP address: %s" % address)
3090 return
0be475ca 3091
b952a52b
MT		 3092+ if not self._check_parsed_network(network):
3093+ return
0be475ca 3094+
b952a52b
MT		 3095 self.db.execute("INSERT INTO networks(network, country) \
3096 VALUES(%s, %s) ON CONFLICT (network) DO \
3097 UPDATE SET country = excluded.country",
3098diff --git a/src/python/location.in b/src/python/location.in
3099index 44ad726..b30beae 100644
3100--- a/src/python/location.in
3101+++ b/src/python/location.in
3102@@ -253,6 +253,7 @@ class CLI(object):
3103 network = db.lookup(address)
3104 except ValueError:
3105 print(_("Invalid IP address: %s") % address, file=sys.stderr)
3106+ return 2
0be475ca 3107
b952a52b
MT		 3108 args = {
3109 "address" : address,
3110@@ -398,10 +399,7 @@ class CLI(object):
0be475ca 3111
b952a52b
MT		 3112 def handle_update(self, db, ns):
3113 if ns.cron and db:
3114- now = datetime.datetime.utcnow()
0be475ca 3115-
b952a52b
MT		 3116- # Parse the database timestamp
3117- t = datetime.datetime.utcfromtimestamp(db.created_at)
3118+ now = time.time()
0be475ca 3119
b952a52b
MT		 3120 if ns.cron == "daily":
3121 delta = datetime.timedelta(days=1)
3122@@ -410,22 +408,20 @@ class CLI(object):
3123 elif ns.cron == "monthly":
3124 delta = datetime.timedelta(days=30)
0be475ca 3125
b952a52b
MT		 3126+ delta = delta.total_seconds()
3127+
3128 # Check if the database has recently been updated
3129- if t >= (now - delta):
3130+ if db.created_at >= (now - delta):
3131 log.info(
3132- _("The database has been updated recently (%s)") % \
3133- format_timedelta(now - t),
3134+ _("The database has been updated recently"),
3135 )
3136 return 3
0be475ca 3137
b952a52b
MT		 3138 # Fetch the timestamp we need from DNS
3139 t = location.discover_latest_version()
0be475ca 3140
b952a52b
MT		 3141- # Parse timestamp into datetime format
3142- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
3143-
3144 # Check the version of the local database
3145- if db and timestamp and db.created_at >= timestamp.timestamp():
3146+ if db and t and db.created_at >= t:
3147 log.info("Already on the latest version")
3148 return
0be475ca 3149
b952a52b 3150@@ -437,7 +433,7 @@ class CLI(object):
0be475ca 3151
b952a52b
MT		 3152 # Try downloading a new database
3153 try:
3154- t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
3155+ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
0be475ca 3156
b952a52b
MT		 3157 # If no file could be downloaded, log a message
3158 except FileNotFoundError as e:
3159@@ -453,13 +449,7 @@ class CLI(object):
0be475ca 3160
b952a52b 3161 return 0
0be475ca 3162
b952a52b
MT		 3163- def handle_verify(self, ns):
3164- try:
3165- db = location.Database(ns.database)
3166- except FileNotFoundError as e:
3167- log.error("%s: %s" % (ns.database, e))
3168- return 127
3169-
3170+ def handle_verify(self, db, ns):
3171 # Verify the database
3172 with open(ns.public_key, "r") as f:
3173 if not db.verify(f):
3174diff --git a/src/python/network.c b/src/python/network.c
4a388bc9 3175index 5496d1e..5b1369d 100644
b952a52b
MT		 3176--- a/src/python/network.c
3177+++ b/src/python/network.c
4a388bc9
MT		 3178@@ -17,13 +17,33 @@
3179 #include <Python.h>
3180
3181 #include <errno.h>
3182+#include <limits.h>
b952a52b
MT		 3183
3184 #include <loc/libloc.h>
3185 #include <loc/network.h>
3186+#include <loc/network-list.h>
3187
3188 #include "locationmodule.h"
3189 #include "network.h"
3190
3191+static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
3192+ PyObject* list = PyList_New(0);
3193+ if (!networks)
3194+ return list;
0be475ca 3195+
b952a52b
MT		 3196+ while (!loc_network_list_empty(networks)) {
3197+ struct loc_network* network = loc_network_list_pop(networks);
0be475ca 3198+
b952a52b
MT		 3199+ PyObject* n = new_network(&NetworkType, network);
3200+ PyList_Append(list, n);
0be475ca 3201+
b952a52b
MT		 3202+ loc_network_unref(network);
3203+ Py_DECREF(n);
0be475ca
MT		 3204+ }
3205+
b952a52b
MT		 3206+ return list;
3207+}
0be475ca 3208+
b952a52b
MT		 3209 PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
3210 NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
3211 if (self) {
4a388bc9
MT		 3212@@ -114,10 +134,18 @@ static int Network_set_asn(NetworkObject* self, PyObject* value) {
3213 long int asn = PyLong_AsLong(value);
3214
3215 // Check if the ASN is within the valid range
3216- if (asn <= 0 || asn > UINT32_MAX) {
3217+ if (asn <= 0) {
3218+ PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn);
3219+ return -1;
3220+ }
3221+
3222+#if (__WORDSIZE > 32)
3223+ // Check whether the input was longer than 32 bit
3224+ if (asn > UINT32_MAX) {
3225 PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn);
3226 return -1;
3227 }
3228+#endif
3229
3230 int r = loc_network_set_asn(self->network, asn);
3231 if (r)
3232@@ -154,13 +182,28 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
b952a52b
MT		 3233 Py_RETURN_NONE;
3234 }
3235
3236+static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
3237+ NetworkObject* other = NULL;
0be475ca 3238+
b952a52b
MT		 3239+ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3240+ return NULL;
0be475ca 3241+
b952a52b 3242+ struct loc_network_list* list = loc_network_exclude(self->network, other->network);
0be475ca 3243+
b952a52b
MT		 3244+ // Convert to Python objects
3245+ PyObject* obj = PyList_FromNetworkList(list);
3246+ loc_network_list_unref(list);
0be475ca 3247+
b952a52b
MT		 3248+ return obj;
3249+}
0be475ca 3250+
b952a52b
MT		 3251 static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
3252 NetworkObject* other = NULL;
0be475ca 3253
b952a52b
MT		 3254 if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3255 return NULL;
0be475ca 3256
b952a52b
MT		 3257- if (loc_network_is_subnet_of(self->network, other->network))
3258+ if (loc_network_is_subnet(other->network, self->network))
3259 Py_RETURN_TRUE;
0be475ca 3260
b952a52b 3261 Py_RETURN_FALSE;
4a388bc9 3262@@ -181,6 +224,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) {
b952a52b 3263 return obj;
0be475ca
MT		 3264 }
3265
b952a52b
MT		 3266+static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) {
3267+ struct in_addr address4;
0be475ca 3268+
b952a52b
MT		 3269+ // Convert IPv4 addresses to struct in_addr
3270+ if (IN6_IS_ADDR_V4MAPPED(address6)) {
3271+ address4.s_addr = address6->s6_addr32[3];
0be475ca 3272+
b952a52b
MT		 3273+ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4));
3274+ }
0be475ca 3275+
b952a52b
MT		 3276+ // Return IPv6 addresses as they are
3277+ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6));
3278+}
0be475ca 3279+
b952a52b
MT		 3280+static PyObject* Network_get__first_address(NetworkObject* self) {
3281+ const struct in6_addr* address = loc_network_get_first_address(self->network);
3282+
3283+ return PyBytes_FromAddress(address);
0be475ca
MT		 3284+}
3285+
b952a52b
MT		 3286 static PyObject* Network_get_last_address(NetworkObject* self) {
3287 char* address = loc_network_format_last_address(self->network);
0be475ca 3288
4a388bc9 3289@@ -190,7 +253,19 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
b952a52b 3290 return obj;
0be475ca 3291 }
0be475ca 3292
b952a52b
MT		 3293+static PyObject* Network_get__last_address(NetworkObject* self) {
3294+ const struct in6_addr* address = loc_network_get_last_address(self->network);
0be475ca 3295+
b952a52b
MT		 3296+ return PyBytes_FromAddress(address);
3297+}
3298+
3299 static struct PyMethodDef Network_methods[] = {
3300+ {
3301+ "exclude",
3302+ (PyCFunction)Network_exclude,
3303+ METH_VARARGS,
3304+ NULL,
3305+ },
3306 {
3307 "has_flag",
3308 (PyCFunction)Network_has_flag,
4a388bc9 3309@@ -241,6 +316,13 @@ static struct PyGetSetDef Network_getsetters[] = {
b952a52b
MT		 3310 NULL,
3311 NULL,
3312 },
3313+ {
3314+ "_first_address",
3315+ (getter)Network_get__first_address,
3316+ NULL,
3317+ NULL,
3318+ NULL,
3319+ },
3320 {
3321 "last_address",
3322 (getter)Network_get_last_address,
4a388bc9 3323@@ -248,6 +330,13 @@ static struct PyGetSetDef Network_getsetters[] = {
b952a52b
MT		 3324 NULL,
3325 NULL,
3326 },
3327+ {
3328+ "_last_address",
3329+ (getter)Network_get__last_address,
3330+ NULL,
3331+ NULL,
3332+ NULL,
3333+ },
3334 { NULL },
3335 };
0be475ca 3336
b952a52b
MT		 3337diff --git a/src/test-as.c b/src/test-as.c
3338index 839a04c..2d61675 100644
3339--- a/src/test-as.c
3340+++ b/src/test-as.c
3341@@ -95,7 +95,7 @@ int main(int argc, char** argv) {
3342 // Enumerator
0be475ca 3343
b952a52b
MT		 3344 struct loc_database_enumerator* enumerator;
3345- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
3346+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
3347 if (err) {
3348 fprintf(stderr, "Could not create a database enumerator\n");
3349 exit(EXIT_FAILURE);
3350diff --git a/src/test-database.c b/src/test-database.c
3351index b4a75c4..da4b11c 100644
3352--- a/src/test-database.c
3353+++ b/src/test-database.c
3354@@ -38,6 +38,14 @@ const char* DESCRIPTION =
3355 "Maecenas ut venenatis nunc.";
3356 const char* LICENSE = "CC";
0be475ca 3357
b952a52b
MT		 3358+const char* networks[] = {
3359+ "2001:db8::/32",
3360+ "2001:db8:1000::/48",
3361+ "2001:db8:2000::/48",
3362+ "2001:db8:2020::/48",
3363+ NULL,
3364+};
3365+
3366 static int attempt_to_open(struct loc_ctx* ctx, char* path) {
3367 FILE* f = fopen(path, "r");
3368 if (!f)
3369@@ -139,6 +147,24 @@ int main(int argc, char** argv) {
3370 exit(EXIT_FAILURE);
0be475ca
MT		 3371 }
3372
b952a52b 3373+ struct loc_network* network = NULL;
0be475ca 3374+
b952a52b
MT		 3375+ // Add some networks
3376+ const char** n = networks;
3377+ while (*n) {
3378+ err = loc_writer_add_network(writer, &network, *n);
3379+ if (err) {
3380+ fprintf(stderr, "Could not add network %s\n", *n);
3381+ exit(EXIT_FAILURE);
3382+ }
0be475ca 3383+
b952a52b
MT		 3384+ // Set a country
3385+ loc_network_set_country_code(network, "XX");
0be475ca 3386+
b952a52b
MT		 3387+ // Next one
3388+ n++;
3389+ }
0be475ca 3390+
b952a52b
MT		 3391 FILE* f = tmpfile();
3392 if (!f) {
3393 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
3394@@ -170,6 +196,33 @@ int main(int argc, char** argv) {
3395 exit(EXIT_FAILURE);
0be475ca
MT		 3396 }
3397
b952a52b
MT		 3398+ // Enumerator
3399+ struct loc_database_enumerator* enumerator;
3400+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
3401+ if (err) {
3402+ fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
3403+ exit(EXIT_FAILURE);
3404+ }
0be475ca 3405+
b952a52b
MT		 3406+ // Walk through all networks
3407+ while (1) {
3408+ err = loc_database_enumerator_next_network(enumerator, &network);
3409+ if (err) {
3410+ fprintf(stderr, "Error fetching the next network: %d\n", err);
3411+ exit(EXIT_FAILURE);
3412+ }
3413+
3414+ if (!network)
3415+ break;
3416+
3417+ char* s = loc_network_str(network);
3418+ printf("Got network: %s\n", s);
3419+ free(s);
3420+ }
3421+
3422+ // Free the enumerator
3423+ loc_database_enumerator_unref(enumerator);
3424+
3425 // Close the database
3426 loc_database_unref(db);
3427 loc_unref(ctx);
0be475ca 3428diff --git a/src/test-network-list.c b/src/test-network-list.c
b952a52b
MT		 3429new file mode 100644
3430index 0000000..6f32ff7
3431--- /dev/null
0be475ca 3432+++ b/src/test-network-list.c
b952a52b
MT		 3433@@ -0,0 +1,183 @@
3434+/*
3435+ libloc - A library to determine the location of someone on the Internet
3436+
3437+ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
3438+
3439+ This program is free software; you can redistribute it and/or modify
3440+ it under the terms of the GNU General Public License as published by
3441+ the Free Software Foundation; either version 2 of the License, or
3442+ (at your option) any later version.
3443+
3444+ This program is distributed in the hope that it will be useful,
3445+ but WITHOUT ANY WARRANTY; without even the implied warranty of
3446+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3447+ GNU General Public License for more details.
3448+*/
3449+
3450+#include <errno.h>
3451+#include <stdio.h>
3452+#include <stddef.h>
3453+#include <stdlib.h>
3454+#include <string.h>
3455+#include <syslog.h>
3456+
3457+#include <loc/libloc.h>
3458+#include <loc/network.h>
3459+#include <loc/network-list.h>
3460+
3461+int main(int argc, char** argv) {
3462+ int err;
3463+
3464+ struct loc_ctx* ctx;
3465+ err = loc_new(&ctx);
3466+ if (err < 0)
3467+ exit(EXIT_FAILURE);
3468+
3469+ // Enable debug logging
3470+ loc_set_log_priority(ctx, LOG_DEBUG);
3471+
3472+ // Create a network
3473+ struct loc_network* network1;
3474+ err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32");
3475+ if (err) {
3476+ fprintf(stderr, "Could not create the network1\n");
3477+ exit(EXIT_FAILURE);
3478+ }
3479+
3480+ struct loc_network* subnet1;
3481+ err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48");
3482+ if (err) {
3483+ fprintf(stderr, "Could not create the subnet1\n");
3484+ exit(EXIT_FAILURE);
3485+ }
3486+
3487+ struct loc_network* subnet2;
3488+ err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48");
3489+ if (err) {
3490+ fprintf(stderr, "Could not create the subnet2\n");
3491+ exit(EXIT_FAILURE);
3492+ }
3493+
0be475ca
MT		 3494+ struct loc_network* subnet3;
3495+ err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48");
3496+ if (err) {
3497+ fprintf(stderr, "Could not create the subnet3\n");
3498+ exit(EXIT_FAILURE);
3499+ }
3500+
3501+ struct loc_network* subnet4;
3502+ err = loc_network_new_from_string(ctx, &subnet4, "2001:db8:d::/48");
3503+ if (err) {
3504+ fprintf(stderr, "Could not create the subnet4\n");
3505+ exit(EXIT_FAILURE);
3506+ }
3507+
b952a52b
MT		 3508+ struct loc_network* subnet5;
3509+ err = loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48");
3510+ if (err) {
3511+ fprintf(stderr, "Could not create the subnet5\n");
3512+ exit(EXIT_FAILURE);
3513+ }
3514+
3515+ struct loc_network* subnet6;
3516+ err = loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48");
3517+ if (err) {
3518+ fprintf(stderr, "Could not create the subnet6\n");
3519+ exit(EXIT_FAILURE);
3520+ }
3521+
3522+ // Make a list with both subnets
3523+ struct loc_network_list* subnets;
3524+ err = loc_network_list_new(ctx, &subnets);
3525+ if (err) {
3526+ fprintf(stderr, "Could not create subnets list\n");
3527+ exit(EXIT_FAILURE);
3528+ }
3529+
3530+ size_t size = loc_network_list_size(subnets);
3531+ if (size > 0) {
3532+ fprintf(stderr, "The list is not empty: %zu\n", size);
3533+ exit(EXIT_FAILURE);
3534+ }
3535+
3536+ err = loc_network_list_push(subnets, subnet1);
3537+ if (err) {
3538+ fprintf(stderr, "Could not add subnet1 to subnets list\n");
3539+ exit(EXIT_FAILURE);
3540+ }
3541+
3542+ if (loc_network_list_empty(subnets)) {
3543+ fprintf(stderr, "The subnets list reports that it is empty\n");
3544+ exit(EXIT_FAILURE);
3545+ }
3546+
3547+ err = loc_network_list_push(subnets, subnet2);
3548+ if (err) {
3549+ fprintf(stderr, "Could not add subnet2 to subnets list\n");
3550+ exit(EXIT_FAILURE);
3551+ }
3552+
3553+ // Add the fourth one next
3554+ err = loc_network_list_push(subnets, subnet4);
3555+ if (err) {
3556+ fprintf(stderr, "Could not add subnet4 to subnets list\n");
3557+ exit(EXIT_FAILURE);
3558+ }
3559+
3560+ // Add the third one
3561+ err = loc_network_list_push(subnets, subnet3);
3562+ if (err) {
3563+ fprintf(stderr, "Could not add subnet3 to subnets list\n");
3564+ exit(EXIT_FAILURE);
3565+ }
3566+
3567+ // Add more subnets
3568+ err = loc_network_list_push(subnets, subnet5);
3569+ if (err) {
3570+ fprintf(stderr, "Could not add subnet5 to subnets list\n");
3571+ exit(EXIT_FAILURE);
3572+ }
3573+
3574+ err = loc_network_list_push(subnets, subnet6);
3575+ if (err) {
3576+ fprintf(stderr, "Could not add subnet6 to subnets list\n");
3577+ exit(EXIT_FAILURE);
3578+ }
3579+
3580+ loc_network_list_dump(subnets);
3581+
3582+ size = loc_network_list_size(subnets);
3583+ if (size != 6) {
3584+ fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
3585+ exit(EXIT_FAILURE);
3586+ }
3587+
3588+ // Exclude subnet1 from network1
3589+ struct loc_network_list* excluded = loc_network_exclude(network1, subnet1);
3590+ if (!excluded) {
3591+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n");
0be475ca
MT		 3592+ exit(EXIT_FAILURE);
3593+ }
3594+
b952a52b
MT		 3595+ loc_network_list_dump(excluded);
3596+
3597+ // Exclude all subnets from network1
3598+ excluded = loc_network_exclude_list(network1, subnets);
3599+ if (!excluded) {
3600+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n");
0be475ca
MT		 3601+ exit(EXIT_FAILURE);
3602+ }
3603+
b952a52b 3604+ loc_network_list_dump(excluded);
0be475ca 3605+
b952a52b
MT		 3606+ if (excluded)
3607+ loc_network_list_unref(excluded);
3608+
3609+ loc_network_list_unref(subnets);
3610+ loc_network_unref(network1);
3611+ loc_network_unref(subnet1);
3612+ loc_network_unref(subnet2);
3613+ loc_unref(ctx);
3614+
3615+ return EXIT_SUCCESS;
3616+}
0be475ca 3617diff --git a/src/test-network.c b/src/test-network.c
b952a52b 3618index d38f13d..dde13f1 100644
0be475ca
MT		 3619--- a/src/test-network.c
3620+++ b/src/test-network.c
3621@@ -14,6 +14,7 @@
3622 GNU General Public License for more details.
3623 */
3624
3625+#include <arpa/inet.h>
3626 #include <errno.h>
3627 #include <stdio.h>
3628 #include <stddef.h>
b952a52b
MT		 3629@@ -37,12 +38,21 @@ int main(int argc, char** argv) {
3630 // Enable debug logging
3631 loc_set_log_priority(ctx, LOG_DEBUG);
0be475ca 3632
b952a52b
MT		 3633+#if 0
3634 struct loc_network_tree* tree;
3635 err = loc_network_tree_new(ctx, &tree);
3636 if (err) {
3637 fprintf(stderr, "Could not create the network tree\n");
3638 exit(EXIT_FAILURE);
3639 }
3640+#endif
3641+
0be475ca
MT		 3642+ struct in6_addr address;
3643+ err = inet_pton(AF_INET6, "2001:db8::1", &address);
3644+ if (err != 1) {
3645+ fprintf(stderr, "Could not parse IP address\n");
3646+ exit(EXIT_FAILURE);
3647+ }
b952a52b 3648
0be475ca
MT		 3649 // Create a network
3650 struct loc_network* network1;
b952a52b 3651@@ -58,12 +68,14 @@ int main(int argc, char** argv) {
0be475ca
MT		 3652 exit(EXIT_FAILURE);
3653 }
3654
b952a52b
MT		 3655+#if 0
3656 // Adding network to the tree
3657 err = loc_network_tree_add_network(tree, network1);
0be475ca 3658 if (err) {
b952a52b
MT		 3659 fprintf(stderr, "Could not add network to the tree\n");
3660 exit(EXIT_FAILURE);
0be475ca 3661 }
b952a52b 3662+#endif
0be475ca 3663
b952a52b
MT		 3664 // Check if the first and last addresses are correct
3665 char* string = loc_network_format_first_address(network1);
3666@@ -88,6 +100,12 @@ int main(int argc, char** argv) {
3667 exit(EXIT_FAILURE);
0be475ca
MT		 3668 }
3669
b952a52b
MT		 3670+ err = loc_network_match_address(network1, &address);
3671+ if (!err) {
3672+ fprintf(stderr, "Network1 does not match address\n");
3673+ exit(EXIT_FAILURE);
3674+ }
0be475ca 3675+
b952a52b
MT		 3676 struct loc_network* network2;
3677 err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48");
3678 if (err) {
3679@@ -101,6 +119,7 @@ int main(int argc, char** argv) {
3680 exit(EXIT_FAILURE);
3681 }
0be475ca 3682
b952a52b
MT		 3683+#if 0
3684 // Adding network to the tree
3685 err = loc_network_tree_add_network(tree, network2);
3686 if (err) {
3687@@ -117,20 +136,84 @@ int main(int argc, char** argv) {
0be475ca 3688
b952a52b
MT		 3689 size_t nodes = loc_network_tree_count_nodes(tree);
3690 printf("The tree has %zu nodes\n", nodes);
3691+#endif
3692+
3693+ // Check equals function
3694+ err = loc_network_cmp(network1, network1);
3695+ if (err) {
3696+ fprintf(stderr, "Network is not equal with itself\n");
3697+ exit(EXIT_FAILURE);
3698+ }
3699+
3700+ err = loc_network_cmp(network1, network2);
3701+ if (!err) {
3702+ fprintf(stderr, "Networks equal unexpectedly\n");
3703+ exit(EXIT_FAILURE);
3704+ }
3705
3706 // Check subnet function
3707- err = loc_network_is_subnet_of(network1, network2);
3708- if (err != 0) {
3709+ err = loc_network_is_subnet(network1, network2);
3710+ if (!err) {
3711 fprintf(stderr, "Subnet check 1 failed: %d\n", err);
3712 exit(EXIT_FAILURE);
0be475ca 3713 }
0be475ca 3714
b952a52b
MT		 3715- err = loc_network_is_subnet_of(network2, network1);
3716- if (err != 1) {
3717+ err = loc_network_is_subnet(network2, network1);
3718+ if (err) {
3719 fprintf(stderr, "Subnet check 2 failed: %d\n", err);
0be475ca
MT		 3720 exit(EXIT_FAILURE);
3721 }
3722
b952a52b
MT		 3723+ // Make subnets
3724+ struct loc_network* subnet1 = NULL;
3725+ struct loc_network* subnet2 = NULL;
3726+
3727+ err = loc_network_subnets(network1, &subnet1, &subnet2);
3728+ if (err || !subnet1 || !subnet2) {
3729+ fprintf(stderr, "Could not find subnets of network: %d\n", err);
3730+ exit(EXIT_FAILURE);
3731+ }
3732+
3733+ char* s = loc_network_str(subnet1);
3734+ printf("Received subnet1 = %s\n", s);
3735+ free(s);
3736+
3737+ s = loc_network_str(subnet2);
3738+ printf("Received subnet2 = %s\n", s);
3739+ free(s);
3740+
3741+ if (!loc_network_is_subnet(network1, subnet1)) {
3742+ fprintf(stderr, "Subnet1 is not a subnet\n");
3743+ exit(EXIT_FAILURE);
3744+ }
3745+
3746+ if (!loc_network_is_subnet(network1, subnet2)) {
3747+ fprintf(stderr, "Subnet2 is not a subnet\n");
3748+ exit(EXIT_FAILURE);
3749+ }
3750+
0be475ca
MT		 3751+ if (!loc_network_overlaps(network1, subnet1)) {
3752+ fprintf(stderr, "Network1 does not seem to contain subnet1\n");
3753+ exit(EXIT_FAILURE);
3754+ }
3755+
3756+ if (!loc_network_overlaps(network1, subnet2)) {
3757+ fprintf(stderr, "Network1 does not seem to contain subnet2\n");
3758+ exit(EXIT_FAILURE);
3759+ }
3760+
b952a52b
MT		 3761+ loc_network_unref(subnet1);
3762+ loc_network_unref(subnet2);
0be475ca 3763+
b952a52b
MT		 3764+ struct loc_network_list* excluded = loc_network_exclude(network1, network2);
3765+ if (!excluded) {
3766+ fprintf(stderr, "Could not create excluded list\n");
3767+ exit(EXIT_FAILURE);
3768+ }
0be475ca 3769+
b952a52b
MT		 3770+ loc_network_list_dump(excluded);
3771+ loc_network_list_unref(excluded);
0be475ca 3772+
b952a52b
MT		 3773 // Create a database
3774 struct loc_writer* writer;
3775 err = loc_writer_new(ctx, &writer, NULL, NULL);
3776@@ -160,6 +243,28 @@ int main(int argc, char** argv) {
3777 // Set ASN
3778 loc_network_set_asn(network4, 1024);
0be475ca 3779
b952a52b
MT		 3780+ // Try adding an invalid network
3781+ struct loc_network* network;
3782+ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
3783+ if (err != -EINVAL) {
3784+ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
3785+ exit(EXIT_FAILURE);
3786+ }
3787+
3788+ // Try adding a single address
3789+ err = loc_writer_add_network(writer, &network, "2001:db8::");
3790+ if (err) {
3791+ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
3792+ exit(EXIT_FAILURE);
3793+ }
3794+
3795+ // Try adding localhost
3796+ err = loc_writer_add_network(writer, &network, "::1/128");
3797+ if (err != -EINVAL) {
3798+ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
3799+ exit(EXIT_FAILURE);
3800+ }
3801+
3802 FILE* f = tmpfile();
3803 if (!f) {
3804 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
3805@@ -177,7 +282,10 @@ int main(int argc, char** argv) {
3806 loc_network_unref(network2);
3807 loc_network_unref(network3);
3808 loc_network_unref(network4);
3809+
3810+#if 0
3811 loc_network_tree_unref(tree);
3812+#endif
0be475ca 3813
b952a52b
MT		 3814 // And open it again from disk
3815 struct loc_database* db;
3816diff --git a/src/writer.c b/src/writer.c
4a388bc9 3817index 5939cff..c61a6df 100644
b952a52b
MT		 3818--- a/src/writer.c
3819+++ b/src/writer.c
3820@@ -147,8 +147,19 @@ static void loc_writer_free(struct loc_writer* writer) {
3821 EVP_PKEY_free(writer->private_key2);
0be475ca 3822
b952a52b
MT		 3823 // Unref all AS
3824- for (unsigned int i = 0; i < writer->as_count; i++) {
3825- loc_as_unref(writer->as[i]);
3826+ if (writer->as) {
3827+ for (unsigned int i = 0; i < writer->as_count; i++) {
3828+ loc_as_unref(writer->as[i]);
3829+ }
3830+ free(writer->as);
3831+ }
0be475ca 3832+
b952a52b
MT		 3833+ // Unref all countries
3834+ if (writer->countries) {
3835+ for (unsigned int i = 0; i < writer->countries_count; i++) {
3836+ loc_country_unref(writer->countries[i]);
3837+ }
3838+ free(writer->countries);
3839 }
0be475ca 3840
b952a52b 3841 // Release network tree
4a388bc9
MT		 3842@@ -601,7 +612,7 @@ static int loc_writer_create_signature(struct loc_writer* writer,
3843 goto END;
3844 }
3845
3846- DEBUG(writer->ctx, "Successfully generated signature of %lu bytes\n", *length);
3847+ DEBUG(writer->ctx, "Successfully generated signature of %zu bytes\n", *length);
3848 r = 0;
3849
3850 // Dump signature
Peter's tree of IPFire 2.x