]> git.ipfire.org Git - people/pmueller/ipfire-2.x.git/blame - src/patches/libloc-0.9.4-upstream.patch
projects / people / pmueller / ipfire-2.x.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
libloc: Import latest changes from upstream
[people/pmueller/ipfire-2.x.git] / src / patches / libloc-0.9.4-upstream.patch
CommitLineData
b952a52b
MT		 1diff --git a/Makefile.am b/Makefile.am
2index a0431a6..ebd7e17 100644
3--- a/Makefile.am
4+++ b/Makefile.am
5@@ -91,11 +91,14 @@ EXTRA_DIST += \
6 pkginclude_HEADERS = \
7 src/loc/libloc.h \
8 src/loc/as.h \
9+ src/loc/as-list.h \
10 src/loc/compat.h \
11 src/loc/country.h \
12+ src/loc/country-list.h \
13 src/loc/database.h \
14 src/loc/format.h \
15 src/loc/network.h \
16+ src/loc/network-list.h \
17 src/loc/private.h \
18 src/loc/stringpool.h \
19 src/loc/resolv.h \
20@@ -107,9 +110,12 @@ lib_LTLIBRARIES = \
21 src_libloc_la_SOURCES = \
22 src/libloc.c \
23 src/as.c \
24+ src/as-list.c \
25 src/country.c \
26+ src/country-list.c \
27 src/database.c \
28 src/network.c \
29+ src/network-list.c \
30 src/resolv.c \
31 src/stringpool.c \
32 src/writer.c
33@@ -312,6 +318,7 @@ check_PROGRAMS = \
34 src/test-database \
35 src/test-as \
36 src/test-network \
37+ src/test-network-list \
38 src/test-country \
39 src/test-signature
05db64d0 40
b952a52b
MT		 41@@ -351,6 +358,15 @@ src_test_network_CFLAGS = \
42 src_test_network_LDADD = \
43 src/libloc.la
05db64d0 44
b952a52b
MT		 45+src_test_network_list_SOURCES = \
46+ src/test-network-list.c
05db64d0 47+
b952a52b
MT		 48+src_test_network_list_CFLAGS = \
49+ $(TESTS_CFLAGS)
05db64d0 50+
b952a52b
MT		 51+src_test_network_list_LDADD = \
52+ src/libloc.la
05db64d0 53+
b952a52b
MT		 54 src_test_stringpool_SOURCES = \
55 src/test-stringpool.c
56
57diff --git a/configure.ac b/configure.ac
58index 2364dfd..012d8ca 100644
59--- a/configure.ac
60+++ b/configure.ac
61@@ -1,6 +1,6 @@
62 AC_PREREQ(2.60)
63 AC_INIT([libloc],
64- [0.9.4],
65+ [0.9.5],
66 [location@lists.ipfire.org],
67 [libloc],
68 [https://location.ipfire.org/])
69diff --git a/src/.gitignore b/src/.gitignore
70index caf80b5..3ccbdb8 100644
71--- a/src/.gitignore
72+++ b/src/.gitignore
73@@ -10,5 +10,6 @@ test-libloc
74 test-database
75 test-country
76 test-network
77+test-network-list
78 test-signature
79 test-stringpool
80diff --git a/src/as-list.c b/src/as-list.c
81new file mode 100644
82index 0000000..5acbb8a
83--- /dev/null
84+++ b/src/as-list.c
85@@ -0,0 +1,161 @@
86+/*
87+ libloc - A library to determine the location of someone on the Internet
05db64d0 88+
b952a52b 89+ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
05db64d0 90+
b952a52b
MT		 91+ This library is free software; you can redistribute it and/or
92+ modify it under the terms of the GNU Lesser General Public
93+ License as published by the Free Software Foundation; either
94+ version 2.1 of the License, or (at your option) any later version.
05db64d0 95+
b952a52b
MT		 96+ This library is distributed in the hope that it will be useful,
97+ but WITHOUT ANY WARRANTY; without even the implied warranty of
98+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
99+ Lesser General Public License for more details.
100+*/
05db64d0 101+
b952a52b
MT		 102+#include <errno.h>
103+#include <stdlib.h>
05db64d0 104+
b952a52b
MT		 105+#include <loc/as.h>
106+#include <loc/as-list.h>
107+#include <loc/private.h>
05db64d0 108+
b952a52b
MT		 109+struct loc_as_list {
110+ struct loc_ctx* ctx;
111+ int refcount;
05db64d0 112+
b952a52b
MT		 113+ struct loc_as** elements;
114+ size_t elements_size;
05db64d0 115+
b952a52b
MT		 116+ size_t size;
117+};
05db64d0 118+
b952a52b
MT		 119+static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
120+ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
121+ list, size, list->elements_size + size);
05db64d0 122+
b952a52b
MT		 123+ struct loc_as** elements = reallocarray(list->elements,
124+ list->elements_size + size, sizeof(*list->elements));
125+ if (!elements)
126+ return -errno;
05db64d0 127+
b952a52b
MT		 128+ list->elements = elements;
129+ list->elements_size += size;
05db64d0 130+
b952a52b
MT		 131+ return 0;
132+}
05db64d0 133+
b952a52b
MT		 134+LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
135+ struct loc_as_list** list) {
136+ struct loc_as_list* l = calloc(1, sizeof(*l));
137+ if (!l)
138+ return -ENOMEM;
05db64d0 139+
b952a52b
MT		 140+ l->ctx = loc_ref(ctx);
141+ l->refcount = 1;
05db64d0 142+
b952a52b
MT		 143+ DEBUG(l->ctx, "AS list allocated at %p\n", l);
144+ *list = l;
05db64d0 145+
b952a52b
MT		 146+ return 0;
147+}
05db64d0 148+
b952a52b
MT		 149+LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
150+ list->refcount++;
05db64d0 151+
b952a52b
MT		 152+ return list;
153+}
05db64d0 154+
b952a52b
MT		 155+static void loc_as_list_free(struct loc_as_list* list) {
156+ DEBUG(list->ctx, "Releasing AS list at %p\n", list);
05db64d0 157+
b952a52b 158+ loc_as_list_clear(list);
05db64d0 159+
b952a52b
MT		 160+ loc_unref(list->ctx);
161+ free(list);
162+}
05db64d0 163+
b952a52b
MT		 164+LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
165+ if (!list)
166+ return NULL;
167+
168+ if (--list->refcount > 0)
169+ return list;
170+
171+ loc_as_list_free(list);
172+ return NULL;
173+}
174+
175+LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
176+ return list->size;
177+}
178+
179+LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
180+ return list->size == 0;
181+}
182+
183+LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
184+ if (!list->elements)
185+ return;
186+
187+ for (unsigned int i = 0; i < list->size; i++)
188+ loc_as_unref(list->elements[i]);
189+
190+ free(list->elements);
191+ list->elements = NULL;
192+ list->elements_size = 0;
193+
194+ list->size = 0;
195+}
196+
197+LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
198+ // Check index
199+ if (index >= list->size)
200+ return NULL;
201+
202+ return loc_as_ref(list->elements[index]);
203+}
204+
205+LOC_EXPORT int loc_as_list_append(
206+ struct loc_as_list* list, struct loc_as* as) {
207+ if (loc_as_list_contains(list, as))
208+ return 0;
209+
210+ // Check if we have space left
211+ if (list->size >= list->elements_size) {
212+ int r = loc_as_list_grow(list, 64);
213+ if (r)
214+ return r;
215+ }
216+
217+ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
218+
219+ list->elements[list->size++] = loc_as_ref(as);
220+
221+ return 0;
222+}
223+
224+LOC_EXPORT int loc_as_list_contains(
225+ struct loc_as_list* list, struct loc_as* as) {
226+ for (unsigned int i = 0; i < list->size; i++) {
227+ if (loc_as_cmp(as, list->elements[i]) == 0)
228+ return 1;
229+ }
230+
231+ return 0;
232+}
233+
234+LOC_EXPORT int loc_as_list_contains_number(
235+ struct loc_as_list* list, uint32_t number) {
236+ struct loc_as* as;
237+
238+ int r = loc_as_new(list->ctx, &as, number);
239+ if (r)
240+ return -1;
241+
242+ r = loc_as_list_contains(list, as);
243+ loc_as_unref(as);
244+
245+ return r;
246+}
05db64d0 247diff --git a/src/as.c b/src/as.c
b952a52b 248index e1fbb01..757bf3d 100644
05db64d0
MT		 249--- a/src/as.c
250+++ b/src/as.c
251@@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
252 }
253
254 LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) {
255- as->name = strdup(name);
256+ if (as->name)
257+ free(as->name);
258+
259+ if (name)
260+ as->name = strdup(name);
261+ else
262+ as->name = NULL;
263
264 return 0;
265 }
b952a52b 266@@ -139,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
05db64d0
MT		 267 if (!string)
268 return 1;
269
270+ // Cannot match anything when name is not set
271+ if (!as->name)
272+ return 1;
273+
274 // Search if string is in name
275 if (strcasestr(as->name, string) != NULL)
276 return 1;
b952a52b
MT		 277diff --git a/src/country-list.c b/src/country-list.c
278new file mode 100644
279index 0000000..cc36740
280--- /dev/null
281+++ b/src/country-list.c
282@@ -0,0 +1,161 @@
283+/*
284+ libloc - A library to determine the location of someone on the Internet
05db64d0 285+
b952a52b 286+ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
05db64d0 287+
b952a52b
MT		 288+ This library is free software; you can redistribute it and/or
289+ modify it under the terms of the GNU Lesser General Public
290+ License as published by the Free Software Foundation; either
291+ version 2.1 of the License, or (at your option) any later version.
05db64d0 292+
b952a52b
MT		 293+ This library is distributed in the hope that it will be useful,
294+ but WITHOUT ANY WARRANTY; without even the implied warranty of
295+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
296+ Lesser General Public License for more details.
297+*/
05db64d0 298+
b952a52b
MT		 299+#include <errno.h>
300+#include <stdlib.h>
05db64d0 301+
b952a52b
MT		 302+#include <loc/country.h>
303+#include <loc/country-list.h>
304+#include <loc/private.h>
05db64d0 305+
b952a52b
MT		 306+struct loc_country_list {
307+ struct loc_ctx* ctx;
308+ int refcount;
05db64d0 309+
b952a52b
MT		 310+ struct loc_country** elements;
311+ size_t elements_size;
05db64d0 312+
b952a52b
MT		 313+ size_t size;
314+};
05db64d0 315+
b952a52b
MT		 316+static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
317+ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
318+ list, size, list->elements_size + size);
05db64d0 319+
b952a52b
MT		 320+ struct loc_country** elements = reallocarray(list->elements,
321+ list->elements_size + size, sizeof(*list->elements));
322+ if (!elements)
323+ return -errno;
05db64d0 324+
b952a52b
MT		 325+ list->elements = elements;
326+ list->elements_size += size;
05db64d0 327+
b952a52b
MT		 328+ return 0;
329+}
05db64d0 330+
b952a52b
MT		 331+LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
332+ struct loc_country_list** list) {
333+ struct loc_country_list* l = calloc(1, sizeof(*l));
334+ if (!l)
335+ return -ENOMEM;
05db64d0 336+
b952a52b
MT		 337+ l->ctx = loc_ref(ctx);
338+ l->refcount = 1;
05db64d0 339+
b952a52b
MT		 340+ DEBUG(l->ctx, "Country list allocated at %p\n", l);
341+ *list = l;
05db64d0 342+
b952a52b
MT		 343+ return 0;
344+}
05db64d0 345+
b952a52b
MT		 346+LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
347+ list->refcount++;
05db64d0 348+
b952a52b
MT		 349+ return list;
350+}
05db64d0 351+
b952a52b
MT		 352+static void loc_country_list_free(struct loc_country_list* list) {
353+ DEBUG(list->ctx, "Releasing country list at %p\n", list);
05db64d0 354+
b952a52b 355+ loc_country_list_clear(list);
05db64d0 356+
b952a52b
MT		 357+ loc_unref(list->ctx);
358+ free(list);
359+}
05db64d0 360+
b952a52b
MT		 361+LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
362+ if (!list)
363+ return NULL;
05db64d0 364+
b952a52b
MT		 365+ if (--list->refcount > 0)
366+ return list;
05db64d0 367+
b952a52b
MT		 368+ loc_country_list_free(list);
369+ return NULL;
370+}
05db64d0 371+
b952a52b
MT		 372+LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
373+ return list->size;
374+}
05db64d0 375+
b952a52b
MT		 376+LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
377+ return list->size == 0;
378+}
05db64d0 379+
b952a52b
MT		 380+LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
381+ if (!list->elements)
382+ return;
05db64d0 383+
b952a52b
MT		 384+ for (unsigned int i = 0; i < list->size; i++)
385+ loc_country_unref(list->elements[i]);
386+
387+ free(list->elements);
388+ list->elements = NULL;
389+ list->elements_size = 0;
390+
391+ list->size = 0;
05db64d0
MT		 392+}
393+
b952a52b
MT		 394+LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
395+ // Check index
396+ if (index >= list->size)
397+ return NULL;
05db64d0 398+
b952a52b
MT		 399+ return loc_country_ref(list->elements[index]);
400+}
05db64d0 401+
b952a52b
MT		 402+LOC_EXPORT int loc_country_list_append(
403+ struct loc_country_list* list, struct loc_country* country) {
404+ if (loc_country_list_contains(list, country))
405+ return 0;
05db64d0 406+
b952a52b
MT		 407+ // Check if we have space left
408+ if (list->size >= list->elements_size) {
409+ int r = loc_country_list_grow(list, 64);
410+ if (r)
411+ return r;
05db64d0
MT		 412+ }
413+
b952a52b 414+ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
05db64d0 415+
b952a52b 416+ list->elements[list->size++] = loc_country_ref(country);
05db64d0 417+
b952a52b
MT		 418+ return 0;
419+}
05db64d0 420+
b952a52b
MT		 421+LOC_EXPORT int loc_country_list_contains(
422+ struct loc_country_list* list, struct loc_country* country) {
423+ for (unsigned int i = 0; i < list->size; i++) {
424+ if (loc_country_cmp(country, list->elements[i]) == 0)
425+ return 1;
426+ }
05db64d0 427+
05db64d0
MT		 428+ return 0;
429+}
430+
b952a52b
MT		 431+LOC_EXPORT int loc_country_list_contains_code(
432+ struct loc_country_list* list, const char* code) {
433+ struct loc_country* country;
05db64d0 434+
b952a52b
MT		 435+ int r = loc_country_new(list->ctx, &country, code);
436+ if (r)
437+ return -1;
05db64d0 438+
b952a52b
MT		 439+ r = loc_country_list_contains(list, country);
440+ loc_country_unref(country);
05db64d0 441+
b952a52b
MT		 442+ return r;
443+}
444diff --git a/src/country.c b/src/country.c
445index 2ba93e6..7aac0db 100644
446--- a/src/country.c
447+++ b/src/country.c
448@@ -34,6 +34,9 @@ struct loc_country {
449 };
05db64d0 450
b952a52b
MT		 451 LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
452+ if (!loc_country_code_is_valid(country_code))
453+ return -EINVAL;
454+
455 struct loc_country* c = calloc(1, sizeof(*c));
456 if (!c)
457 return -ENOMEM;
05db64d0 458diff --git a/src/database.c b/src/database.c
b952a52b 459index fa1dad0..1871b74 100644
05db64d0
MT		 460--- a/src/database.c
461+++ b/src/database.c
b952a52b 462@@ -38,8 +38,10 @@
05db64d0 463
b952a52b
MT		 464 #include <loc/libloc.h>
465 #include <loc/as.h>
466+#include <loc/as-list.h>
467 #include <loc/compat.h>
468 #include <loc/country.h>
469+#include <loc/country-list.h>
470 #include <loc/database.h>
471 #include <loc/format.h>
472 #include <loc/network.h>
473@@ -99,11 +101,14 @@ struct loc_database_enumerator {
05db64d0 474
b952a52b
MT		 475 // Search string
476 char* string;
477- char country_code[3];
478- uint32_t asn;
479+ struct loc_country_list* countries;
480+ struct loc_as_list* asns;
481 enum loc_network_flags flags;
482 int family;
05db64d0 483
b952a52b
MT		 484+ // Flatten output?
485+ int flatten;
486+
487 // Index of the AS we are looking at
488 unsigned int as_index;
05db64d0 489
b952a52b
MT		 490@@ -115,6 +120,9 @@ struct loc_database_enumerator {
491 struct loc_node_stack network_stack[MAX_STACK_DEPTH];
492 int network_stack_depth;
493 unsigned int* networks_visited;
05db64d0 494+
b952a52b
MT		 495+ // For subnet search
496+ struct loc_network_list* stack;
497 };
05db64d0 498
b952a52b
MT		 499 static int loc_database_read_magic(struct loc_database* db) {
500@@ -611,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
05db64d0
MT		 501 }
502
b952a52b
MT		 503 clock_t end = clock();
504- DEBUG(db->ctx, "Signature checked in %.4fms\n",
505+ INFO(db->ctx, "Signature checked in %.4fms\n",
506 (double)(end - start) / CLOCKS_PER_SEC * 1000);
05db64d0 507
b952a52b
MT		 508 CLEANUP:
509@@ -671,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
510 off_t lo = 0;
511 off_t hi = db->as_count - 1;
05db64d0 512
b952a52b
MT		 513+#ifdef ENABLE_DEBUG
514 // Save start time
515 clock_t start = clock();
516+#endif
05db64d0 517
b952a52b
MT		 518 while (lo <= hi) {
519 off_t i = (lo + hi) / 2;
520@@ -685,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
521 // Check if this is a match
522 uint32_t as_number = loc_as_get_number(*as);
523 if (as_number == number) {
524+#ifdef ENABLE_DEBUG
525 clock_t end = clock();
05db64d0 526
b952a52b
MT		 527 // Log how fast this has been
528 DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
529 (double)(end - start) / CLOCKS_PER_SEC * 1000);
530+#endif
05db64d0 531
b952a52b
MT		 532 return 0;
533 }
534@@ -733,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
535 return -1;
536 }
05db64d0 537
b952a52b
MT		 538+#ifdef ENABLE_DEBUG
539 if (r == 0) {
540 char* string = loc_network_str(*network);
541 DEBUG(db->ctx, "Got network %s\n", string);
542 free(string);
543 }
544+#endif
05db64d0 545
b952a52b 546 return r;
05db64d0 547 }
b952a52b
MT		 548@@ -762,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru
549 }
05db64d0 550
b952a52b
MT		 551 // Check if the given IP address is inside the network
552- r = loc_network_match_address(*network, address);
553- if (r) {
554+ if (!loc_network_match_address(*network, address)) {
555 DEBUG(db->ctx, "Searched address is not part of the network\n");
05db64d0 556
b952a52b
MT		 557 loc_network_unref(*network);
558@@ -832,17 +845,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
05db64d0 559
b952a52b 560 *network = NULL;
05db64d0 561
b952a52b
MT		 562+#ifdef ENABLE_DEBUG
563 // Save start time
564 clock_t start = clock();
565+#endif
05db64d0 566
b952a52b
MT		 567 int r = __loc_database_lookup(db, address, network, &network_address,
568 db->network_nodes_v1, 0);
05db64d0 569
b952a52b
MT		 570+#ifdef ENABLE_DEBUG
571 clock_t end = clock();
05db64d0 572
b952a52b
MT		 573 // Log how fast this has been
574 DEBUG(db->ctx, "Executed network search in %.4fms\n",
575 (double)(end - start) / CLOCKS_PER_SEC * 1000);
576+#endif
577
578 return r;
05db64d0 579 }
b952a52b
MT		 580@@ -889,8 +906,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
581 off_t lo = 0;
582 off_t hi = db->countries_count - 1;
05db64d0 583
b952a52b
MT		 584+#ifdef ENABLE_DEBUG
585 // Save start time
586 clock_t start = clock();
587+#endif
588
589 while (lo <= hi) {
590 off_t i = (lo + hi) / 2;
591@@ -905,11 +924,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
592 int result = strcmp(code, cc);
593
594 if (result == 0) {
595+#ifdef ENABLE_DEBUG
596 clock_t end = clock();
597
598 // Log how fast this has been
599 DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
600 (double)(end - start) / CLOCKS_PER_SEC * 1000);
601+#endif
602
603 return 0;
604 }
605@@ -932,8 +953,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
606
607 // Enumerator
608
609+static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
610+ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
611+
612+ // Release all references
613+ loc_database_unref(enumerator->db);
614+ loc_unref(enumerator->ctx);
615+
616+ if (enumerator->string)
617+ free(enumerator->string);
618+
619+ if (enumerator->countries)
620+ loc_country_list_unref(enumerator->countries);
621+
622+ if (enumerator->asns)
623+ loc_as_list_unref(enumerator->asns);
624+
625+ // Free network search
626+ free(enumerator->networks_visited);
627+
628+ // Free subnet stack
629+ if (enumerator->stack)
630+ loc_network_list_unref(enumerator->stack);
631+
632+ free(enumerator);
633+}
634+
635 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
636- struct loc_database* db, enum loc_database_enumerator_mode mode) {
637+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
638 struct loc_database_enumerator* e = calloc(1, sizeof(*e));
639 if (!e)
640 return -ENOMEM;
641@@ -944,11 +991,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
642 e->mode = mode;
643 e->refcount = 1;
644
645+ // Flatten output?
646+ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
647+
648 // Initialise graph search
649- //e->network_stack[++e->network_stack_depth] = 0;
650 e->network_stack_depth = 1;
651 e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
05db64d0 652
b952a52b
MT		 653+ // Allocate stack
654+ int r = loc_network_list_new(e->ctx, &e->stack);
655+ if (r) {
656+ loc_database_enumerator_free(e);
657+ return r;
658+ }
659+
660 DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
05db64d0 661
b952a52b
MT		 662 *enumerator = e;
663@@ -961,22 +1017,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
664 return enumerator;
05db64d0
MT		 665 }
666
b952a52b
MT		 667-static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
668- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
669-
670- // Release all references
671- loc_database_unref(enumerator->db);
672- loc_unref(enumerator->ctx);
673-
674- if (enumerator->string)
675- free(enumerator->string);
676-
677- // Free network search
678- free(enumerator->networks_visited);
679-
680- free(enumerator);
681-}
682-
683 LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
684 if (!enumerator)
685 return NULL;
686@@ -998,40 +1038,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
05db64d0
MT		 687 return 0;
688 }
689
b952a52b
MT		 690-LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
691- // Set empty country code
692- if (!country_code || !*country_code) {
693- *enumerator->country_code = '\0';
694- return 0;
695- }
696+LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
697+ struct loc_database_enumerator* enumerator) {
698+ if (!enumerator->countries)
699+ return NULL;
05db64d0 700
b952a52b
MT		 701- // Treat A1, A2, A3 as special country codes,
702- // but perform search for flags instead
703- if (strcmp(country_code, "A1") == 0) {
704- return loc_database_enumerator_set_flag(enumerator,
705- LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
706- } else if (strcmp(country_code, "A2") == 0) {
707- return loc_database_enumerator_set_flag(enumerator,
708- LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
709- } else if (strcmp(country_code, "A3") == 0) {
710- return loc_database_enumerator_set_flag(enumerator,
711- LOC_NETWORK_FLAG_ANYCAST);
712- }
713+ return loc_country_list_ref(enumerator->countries);
714+}
05db64d0 715
b952a52b
MT		 716- // Country codes must be two characters
717- if (!loc_country_code_is_valid(country_code))
718- return -EINVAL;
719+LOC_EXPORT int loc_database_enumerator_set_countries(
720+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
721+ if (enumerator->countries)
722+ loc_country_list_unref(enumerator->countries);
05db64d0 723
b952a52b
MT		 724- for (unsigned int i = 0; i < 3; i++) {
725- enumerator->country_code[i] = country_code[i];
726- }
727+ enumerator->countries = loc_country_list_ref(countries);
05db64d0 728
b952a52b 729 return 0;
05db64d0
MT		 730 }
731
b952a52b
MT		 732-LOC_EXPORT int loc_database_enumerator_set_asn(
733- struct loc_database_enumerator* enumerator, unsigned int asn) {
734- enumerator->asn = asn;
735+LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
736+ struct loc_database_enumerator* enumerator) {
737+ if (!enumerator->asns)
738+ return NULL;
739+
740+ return loc_as_list_ref(enumerator->asns);
741+}
742+
743+LOC_EXPORT int loc_database_enumerator_set_asns(
744+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
745+ if (enumerator->asns)
746+ loc_as_list_unref(enumerator->asns);
747+
748+ enumerator->asns = loc_as_list_ref(asns);
05db64d0 749
b952a52b
MT		 750 return 0;
751 }
752@@ -1110,16 +1148,64 @@ static int loc_database_enumerator_stack_push_node(
753 return 0;
05db64d0
MT		 754 }
755
b952a52b
MT		 756-LOC_EXPORT int loc_database_enumerator_next_network(
757- struct loc_database_enumerator* enumerator, struct loc_network** network) {
758- // Reset network
759- *network = NULL;
760+static int loc_database_enumerator_filter_network(
761+ struct loc_database_enumerator* enumerator, struct loc_network* network) {
762+ // Skip if the family does not match
763+ if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
764+ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
765+ return 1;
766+ }
05db64d0 767
b952a52b
MT		 768- // Do not do anything if not in network mode
769- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
770- return 0;
771+ // Skip if the country code does not match
772+ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
773+ const char* country_code = loc_network_get_country_code(network);
05db64d0 774
b952a52b
MT		 775- int r;
776+ if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
777+ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
778+ return 1;
779+ }
780+ }
781+
782+ // Skip if the ASN does not match
783+ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
784+ uint32_t asn = loc_network_get_asn(network);
785+
786+ if (!loc_as_list_contains_number(enumerator->asns, asn)) {
787+ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
788+ return 1;
789+ }
790+ }
791+
792+ // Skip if flags do not match
793+ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
794+ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
795+ return 1;
796+ }
797+
798+ // Do not filter
799+ return 0;
800+}
801+
802+static int __loc_database_enumerator_next_network(
803+ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
804+ // Return top element from the stack
805+ while (1) {
806+ *network = loc_network_list_pop_first(enumerator->stack);
807+
808+ // Stack is empty
809+ if (!*network)
810+ break;
811+
812+ // Throw away any networks by filter
813+ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
814+ loc_network_unref(*network);
815+ *network = NULL;
816+ continue;
817+ }
818+
819+ // Return result
820+ return 0;
821+ }
05db64d0 822
b952a52b
MT		 823 DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
824 enumerator->network_stack_depth);
825@@ -1149,7 +1235,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
826 enumerator->db->network_nodes_v1 + node->offset;
05db64d0 827
b952a52b
MT		 828 // Add edges to stack
829- r = loc_database_enumerator_stack_push_node(enumerator,
830+ int r = loc_database_enumerator_stack_push_node(enumerator,
831 be32toh(n->one), 1, node->depth + 1);
05db64d0 832
b952a52b
MT		 833 if (r)
834@@ -1175,56 +1261,145 @@ LOC_EXPORT int loc_database_enumerator_next_network(
835 if (r)
836 return r;
05db64d0 837
b952a52b
MT		 838- // Check if we are interested in this network
839+ // Return all networks when the filter is disabled
840+ if (!filter)
841+ return 0;
05db64d0 842
b952a52b
MT		 843- // Skip if the family does not match
844- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
845+ // Check if we are interested in this network
846+ if (loc_database_enumerator_filter_network(enumerator, *network)) {
847 loc_network_unref(*network);
848 *network = NULL;
05db64d0 849
b952a52b
MT		 850 continue;
851 }
05db64d0 852
b952a52b
MT		 853- // Skip if the country code does not match
854- if (*enumerator->country_code &&
855- !loc_network_match_country_code(*network, enumerator->country_code)) {
856- loc_network_unref(*network);
857- *network = NULL;
858+ return 0;
859+ }
860+ }
05db64d0 861
b952a52b
MT		 862- continue;
863- }
864+ // Reached the end of the search
865+ return 0;
866+}
05db64d0 867
b952a52b
MT		 868- // Skip if the ASN does not match
869- if (enumerator->asn &&
870- !loc_network_match_asn(*network, enumerator->asn)) {
871- loc_network_unref(*network);
872- *network = NULL;
873+static int __loc_database_enumerator_next_network_flattened(
874+ struct loc_database_enumerator* enumerator, struct loc_network** network) {
875+ // Fetch the next network
876+ int r = __loc_database_enumerator_next_network(enumerator, network, 1);
877+ if (r)
878+ return r;
05db64d0 879
b952a52b
MT		 880- continue;
881- }
882+ // End if we could not read another network
883+ if (!*network)
884+ return 0;
05db64d0 885
b952a52b
MT		 886- // Skip if flags do not match
887- if (enumerator->flags &&
888- !loc_network_match_flag(*network, enumerator->flags)) {
889- loc_network_unref(*network);
890- *network = NULL;
891+ struct loc_network* subnet = NULL;
892+ struct loc_network_list* subnets;
05db64d0 893
b952a52b
MT		 894- continue;
895+ // Create a list with all subnets
896+ r = loc_network_list_new(enumerator->ctx, &subnets);
897+ if (r)
898+ return r;
899+
900+ // Search all subnets from the database
901+ while (1) {
902+ // Fetch the next network in line
903+ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
904+ if (r) {
905+ loc_network_unref(subnet);
906+ loc_network_list_unref(subnets);
907+
908+ return r;
909+ }
910+
911+ // End if we did not receive another subnet
912+ if (!subnet)
913+ break;
914+
915+ // Collect all subnets in a list
916+ if (loc_network_is_subnet(*network, subnet)) {
917+ r = loc_network_list_push(subnets, subnet);
918+ if (r) {
919+ loc_network_unref(subnet);
920+ loc_network_list_unref(subnets);
921+
922+ return r;
923 }
05db64d0 924
b952a52b
MT		 925- return 0;
926+ loc_network_unref(subnet);
927+ continue;
928 }
929+
930+ // If this is not a subnet, we push it back onto the stack and break
931+ r = loc_network_list_push(enumerator->stack, subnet);
932+ if (r) {
933+ loc_network_unref(subnet);
934+ loc_network_list_unref(subnets);
935+
936+ return r;
937+ }
938+
939+ loc_network_unref(subnet);
940+ break;
941 }
05db64d0 942
b952a52b
MT		 943- // Reached the end of the search
944+ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
945+
946+ // We can abort here if the network has no subnets
947+ if (loc_network_list_empty(subnets)) {
948+ loc_network_list_unref(subnets);
949+
950+ return 0;
951+ }
05db64d0 952
b952a52b
MT		 953- // Mark all nodes as non-visited
954- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
955- enumerator->networks_visited[i] = 0;
956+ // If the network has any subnets, we will break it into smaller parts
957+ // without the subnets.
958+ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
959+ if (!excluded) {
960+ loc_network_list_unref(subnets);
961+ return -1;
962+ }
963+
964+ // Merge subnets onto the stack
965+ r = loc_network_list_merge(enumerator->stack, subnets);
966+ if (r) {
967+ loc_network_list_unref(subnets);
968+ loc_network_list_unref(excluded);
969+
970+ return r;
971+ }
972+
973+ // Push excluded list onto the stack
974+ r = loc_network_list_merge(enumerator->stack, excluded);
975+ if (r) {
976+ loc_network_list_unref(subnets);
977+ loc_network_list_unref(excluded);
978+
979+ return r;
980+ }
981+
982+ loc_network_list_unref(subnets);
983+ loc_network_list_unref(excluded);
984+
985+ // Replace network with the first one from the stack
986+ loc_network_unref(*network);
987+ *network = loc_network_list_pop_first(enumerator->stack);
05db64d0 988
b952a52b
MT		 989 return 0;
990 }
05db64d0 991
b952a52b
MT		 992+LOC_EXPORT int loc_database_enumerator_next_network(
993+ struct loc_database_enumerator* enumerator, struct loc_network** network) {
994+ // Do not do anything if not in network mode
995+ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
996+ return 0;
997+
998+ // Flatten output?
999+ if (enumerator->flatten)
1000+ return __loc_database_enumerator_next_network_flattened(enumerator, network);
1001+
1002+ return __loc_database_enumerator_next_network(enumerator, network, 1);
1003+}
1004+
1005 LOC_EXPORT int loc_database_enumerator_next_country(
1006 struct loc_database_enumerator* enumerator, struct loc_country** country) {
1007 *country = NULL;
05db64d0 1008diff --git a/src/libloc.sym b/src/libloc.sym
b952a52b 1009index b8296eb..ee333f1 100644
05db64d0
MT		 1010--- a/src/libloc.sym
1011+++ b/src/libloc.sym
b952a52b
MT		 1012@@ -37,6 +37,18 @@ global:
1013 loc_as_set_name;
1014 loc_as_unref;
1015
1016+ # AS List
1017+ loc_as_list_append;
1018+ loc_as_list_clear;
1019+ loc_as_list_contains;
1020+ loc_as_list_contains_number;
1021+ loc_as_list_empty;
1022+ loc_as_list_get;
1023+ loc_as_list_new;
1024+ loc_as_list_ref;
1025+ loc_as_list_size;
1026+ loc_as_list_unref;
1027+
1028 # Country
1029 loc_country_cmp;
1030 loc_country_code_is_valid;
1031@@ -49,6 +61,18 @@ global:
1032 loc_country_set_name;
1033 loc_country_unref;
1034
1035+ # Country List
1036+ loc_country_list_append;
1037+ loc_country_list_clear;
1038+ loc_country_list_contains;
1039+ loc_country_list_contains_code;
1040+ loc_country_list_empty;
1041+ loc_country_list_get;
1042+ loc_country_list_new;
1043+ loc_country_list_ref;
1044+ loc_country_list_size;
1045+ loc_country_list_unref;
1046+
1047 # Database
1048 loc_database_add_as;
1049 loc_database_count_as;
1050@@ -66,13 +90,15 @@ global:
1051 loc_database_verify;
1052
1053 # Database Enumerator
1054+ loc_database_enumerator_get_asns;
1055+ loc_database_enumerator_get_countries;
1056 loc_database_enumerator_new;
1057 loc_database_enumerator_next_as;
1058 loc_database_enumerator_next_country;
1059 loc_database_enumerator_next_network;
1060 loc_database_enumerator_ref;
1061- loc_database_enumerator_set_asn;
1062- loc_database_enumerator_set_country_code;
1063+ loc_database_enumerator_set_asns;
1064+ loc_database_enumerator_set_countries;
1065 loc_database_enumerator_set_family;
1066 loc_database_enumerator_set_flag;
1067 loc_database_enumerator_set_string;
1068@@ -80,24 +106,48 @@ global:
1069
1070 # Network
1071 loc_network_address_family;
1072+ loc_network_cmp;
1073+ loc_network_exclude;
1074+ loc_network_exclude_list;
1075 loc_network_format_first_address;
05db64d0
MT		 1076 loc_network_format_last_address;
1077 loc_network_get_asn;
1078 loc_network_get_country_code;
b952a52b
MT		 1079+ loc_network_get_first_address;
1080+ loc_network_get_last_address;
05db64d0 1081 loc_network_has_flag;
b952a52b
MT		 1082- loc_network_is_subnet_of;
1083+ loc_network_is_subnet;
1084+ loc_network_match_address;
1085 loc_network_match_asn;
1086 loc_network_match_country_code;
1087 loc_network_match_flag;
1088 loc_network_new;
1089 loc_network_new_from_string;
1090+ loc_network_overlaps;
1091+ loc_network_prefix;
1092 loc_network_ref;
1093 loc_network_set_asn;
1094 loc_network_set_country_code;
1095 loc_network_set_flag;
1096 loc_network_str;
1097+ loc_network_subnets;
1098 loc_network_unref;
1099
1100+ # Network List
1101+ loc_network_list_clear;
1102+ loc_network_list_contains;
1103+ loc_network_list_dump;
1104+ loc_network_list_empty;
1105+ loc_network_list_get;
1106+ loc_network_list_merge;
1107+ loc_network_list_new;
1108+ loc_network_list_pop;
1109+ loc_network_list_pop_first;
1110+ loc_network_list_push;
1111+ loc_network_list_ref;
1112+ loc_network_list_size;
1113+ loc_network_list_unref;
1114+
1115 # Writer
1116 loc_writer_add_as;
1117 loc_writer_add_country;
1118diff --git a/src/loc/as-list.h b/src/loc/as-list.h
05db64d0 1119new file mode 100644
b952a52b 1120index 0000000..7b5c4e8
05db64d0 1121--- /dev/null
b952a52b
MT		 1122+++ b/src/loc/as-list.h
1123@@ -0,0 +1,41 @@
05db64d0
MT		 1124+/*
1125+ libloc - A library to determine the location of someone on the Internet
1126+
b952a52b 1127+ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
05db64d0
MT		 1128+
1129+ This library is free software; you can redistribute it and/or
1130+ modify it under the terms of the GNU Lesser General Public
1131+ License as published by the Free Software Foundation; either
1132+ version 2.1 of the License, or (at your option) any later version.
1133+
1134+ This library is distributed in the hope that it will be useful,
1135+ but WITHOUT ANY WARRANTY; without even the implied warranty of
1136+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1137+ Lesser General Public License for more details.
1138+*/
1139+
b952a52b
MT		 1140+#ifndef LIBLOC_AS_LIST_H
1141+#define LIBLOC_AS_LIST_H
05db64d0 1142+
b952a52b
MT		 1143+#include <loc/as.h>
1144+#include <loc/libloc.h>
1145+
1146+struct loc_as_list;
1147+
1148+int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
1149+struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
1150+struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
1151+
1152+size_t loc_as_list_size(struct loc_as_list* list);
1153+int loc_as_list_empty(struct loc_as_list* list);
1154+void loc_as_list_clear(struct loc_as_list* list);
1155+
1156+struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
1157+int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
1158+
1159+int loc_as_list_contains(
1160+ struct loc_as_list* list, struct loc_as* as);
1161+int loc_as_list_contains_number(
1162+ struct loc_as_list* list, uint32_t number);
05db64d0
MT		 1163+
1164+#endif
b952a52b 1165diff --git a/src/loc/country-list.h b/src/loc/country-list.h
05db64d0 1166new file mode 100644
b952a52b 1167index 0000000..a7f818a
05db64d0 1168--- /dev/null
b952a52b
MT		 1169+++ b/src/loc/country-list.h
1170@@ -0,0 +1,43 @@
05db64d0
MT		 1171+/*
1172+ libloc - A library to determine the location of someone on the Internet
1173+
b952a52b 1174+ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
05db64d0
MT		 1175+
1176+ This library is free software; you can redistribute it and/or
1177+ modify it under the terms of the GNU Lesser General Public
1178+ License as published by the Free Software Foundation; either
1179+ version 2.1 of the License, or (at your option) any later version.
1180+
1181+ This library is distributed in the hope that it will be useful,
1182+ but WITHOUT ANY WARRANTY; without even the implied warranty of
b952a52b
MT		 1183+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1184+ Lesser General Public License for more details.
1185+*/
05db64d0 1186+
b952a52b
MT		 1187+#ifndef LIBLOC_COUNTRY_LIST_H
1188+#define LIBLOC_COUNTRY_LIST_H
05db64d0 1189+
b952a52b 1190+#include <stdlib.h>
05db64d0 1191+
b952a52b
MT		 1192+#include <loc/libloc.h>
1193+#include <loc/country.h>
05db64d0 1194+
b952a52b 1195+struct loc_country_list;
05db64d0 1196+
b952a52b
MT		 1197+int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
1198+struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
1199+struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
05db64d0 1200+
b952a52b
MT		 1201+size_t loc_country_list_size(struct loc_country_list* list);
1202+int loc_country_list_empty(struct loc_country_list* list);
1203+void loc_country_list_clear(struct loc_country_list* list);
05db64d0 1204+
b952a52b
MT		 1205+struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
1206+int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
05db64d0 1207+
b952a52b
MT		 1208+int loc_country_list_contains(
1209+ struct loc_country_list* list, struct loc_country* country);
1210+int loc_country_list_contains_code(
1211+ struct loc_country_list* list, const char* code);
05db64d0 1212+
b952a52b
MT		 1213+#endif
1214diff --git a/src/loc/database.h b/src/loc/database.h
1215index 43173dd..70801f0 100644
1216--- a/src/loc/database.h
1217+++ b/src/loc/database.h
1218@@ -25,6 +25,7 @@
1219 #include <loc/network.h>
1220 #include <loc/as.h>
1221 #include <loc/country.h>
1222+#include <loc/country-list.h>
1223
1224 struct loc_database;
1225 int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
1226@@ -55,15 +56,24 @@ enum loc_database_enumerator_mode {
1227 LOC_DB_ENUMERATE_COUNTRIES = 3,
1228 };
1229
1230+enum loc_database_enumerator_flags {
1231+ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
1232+};
05db64d0 1233+
b952a52b
MT		 1234 struct loc_database_enumerator;
1235 int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
1236- struct loc_database* db, enum loc_database_enumerator_mode mode);
1237+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
1238 struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
1239 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
1240
1241 int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
1242-int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
1243-int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
1244+struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
1245+int loc_database_enumerator_set_countries(
1246+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
1247+struct loc_as_list* loc_database_enumerator_get_asns(
1248+ struct loc_database_enumerator* enumerator);
1249+int loc_database_enumerator_set_asns(
1250+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
1251 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
1252 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
1253 int loc_database_enumerator_next_as(
1254diff --git a/src/loc/network-list.h b/src/loc/network-list.h
1255new file mode 100644
1256index 0000000..bee21c4
1257--- /dev/null
1258+++ b/src/loc/network-list.h
1259@@ -0,0 +1,37 @@
1260+/*
1261+ libloc - A library to determine the location of someone on the Internet
05db64d0 1262+
b952a52b 1263+ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
05db64d0 1264+
b952a52b
MT		 1265+ This library is free software; you can redistribute it and/or
1266+ modify it under the terms of the GNU Lesser General Public
1267+ License as published by the Free Software Foundation; either
1268+ version 2.1 of the License, or (at your option) any later version.
05db64d0 1269+
b952a52b
MT		 1270+ This library is distributed in the hope that it will be useful,
1271+ but WITHOUT ANY WARRANTY; without even the implied warranty of
1272+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1273+ Lesser General Public License for more details.
1274+*/
05db64d0 1275+
b952a52b
MT		 1276+#ifndef LIBLOC_NETWORK_LIST_H
1277+#define LIBLOC_NETWORK_LIST_H
05db64d0 1278+
b952a52b 1279+#include <loc/network.h>
05db64d0 1280+
b952a52b
MT		 1281+struct loc_network_list;
1282+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
1283+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
1284+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
1285+size_t loc_network_list_size(struct loc_network_list* list);
1286+int loc_network_list_empty(struct loc_network_list* list);
1287+void loc_network_list_clear(struct loc_network_list* list);
1288+void loc_network_list_dump(struct loc_network_list* list);
1289+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
1290+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
1291+struct loc_network* loc_network_list_pop(struct loc_network_list* list);
1292+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
1293+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
1294+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
05db64d0 1295+
b952a52b
MT		 1296+#endif
1297diff --git a/src/loc/network.h b/src/loc/network.h
1298index 70c3803..af3dafd 100644
1299--- a/src/loc/network.h
1300+++ b/src/loc/network.h
1301@@ -21,6 +21,7 @@
1302
1303 #include <loc/libloc.h>
1304 #include <loc/format.h>
05db64d0 1305+#include <loc/network-list.h>
05db64d0 1306
b952a52b
MT		 1307 enum loc_network_flags {
1308 LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
1309@@ -37,8 +38,11 @@ struct loc_network* loc_network_ref(struct loc_network* network);
1310 struct loc_network* loc_network_unref(struct loc_network* network);
1311 char* loc_network_str(struct loc_network* network);
1312 int loc_network_address_family(struct loc_network* network);
1313+unsigned int loc_network_prefix(struct loc_network* network);
05db64d0 1314
b952a52b
MT		 1315+const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
1316 char* loc_network_format_first_address(struct loc_network* network);
1317+const struct in6_addr* loc_network_get_last_address(struct loc_network* network);
1318 char* loc_network_format_last_address(struct loc_network* network);
1319 int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
05db64d0 1320
b952a52b
MT		 1321@@ -54,7 +58,14 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
1322 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
1323 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
1324
1325-int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
1326+int loc_network_cmp(struct loc_network* self, struct loc_network* other);
1327+int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
1328+int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
1329+int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
1330+struct loc_network_list* loc_network_exclude(
1331+ struct loc_network* self, struct loc_network* other);
1332+struct loc_network_list* loc_network_exclude_list(
1333+ struct loc_network* network, struct loc_network_list* list);
1334
1335 #ifdef LIBLOC_PRIVATE
05db64d0 1336
b952a52b 1337diff --git a/src/network-list.c b/src/network-list.c
05db64d0 1338new file mode 100644
b952a52b 1339index 0000000..e2b20f3
05db64d0 1340--- /dev/null
b952a52b
MT		 1341+++ b/src/network-list.c
1342@@ -0,0 +1,295 @@
05db64d0
MT		 1343+/*
1344+ libloc - A library to determine the location of someone on the Internet
1345+
1346+ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
1347+
1348+ This library is free software; you can redistribute it and/or
1349+ modify it under the terms of the GNU Lesser General Public
1350+ License as published by the Free Software Foundation; either
1351+ version 2.1 of the License, or (at your option) any later version.
1352+
1353+ This library is distributed in the hope that it will be useful,
1354+ but WITHOUT ANY WARRANTY; without even the implied warranty of
1355+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1356+ Lesser General Public License for more details.
1357+*/
1358+
1359+#include <errno.h>
1360+#include <stdlib.h>
b952a52b 1361+#include <time.h>
05db64d0 1362+
b952a52b
MT		 1363+#include <loc/libloc.h>
1364+#include <loc/network.h>
05db64d0
MT		 1365+#include <loc/private.h>
1366+
b952a52b 1367+struct loc_network_list {
05db64d0
MT		 1368+ struct loc_ctx* ctx;
1369+ int refcount;
1370+
b952a52b
MT		 1371+ struct loc_network** elements;
1372+ size_t elements_size;
1373+
05db64d0 1374+ size_t size;
05db64d0
MT		 1375+};
1376+
b952a52b
MT		 1377+static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
1378+ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
1379+ list, size, list->elements_size + size);
1380+
1381+ struct loc_network** elements = reallocarray(list->elements,
1382+ list->elements_size + size, sizeof(*list->elements));
1383+ if (!elements)
1384+ return -errno;
1385+
1386+ list->elements = elements;
1387+ list->elements_size += size;
1388+
1389+ return 0;
1390+}
1391+
1392+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
1393+ struct loc_network_list** list) {
1394+ struct loc_network_list* l = calloc(1, sizeof(*l));
05db64d0
MT		 1395+ if (!l)
1396+ return -ENOMEM;
1397+
1398+ l->ctx = loc_ref(ctx);
1399+ l->refcount = 1;
1400+
b952a52b 1401+ DEBUG(l->ctx, "Network list allocated at %p\n", l);
05db64d0 1402+ *list = l;
05db64d0
MT		 1403+ return 0;
1404+}
1405+
b952a52b 1406+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
05db64d0
MT		 1407+ list->refcount++;
1408+
1409+ return list;
1410+}
1411+
b952a52b
MT		 1412+static void loc_network_list_free(struct loc_network_list* list) {
1413+ DEBUG(list->ctx, "Releasing network list at %p\n", list);
05db64d0 1414+
b952a52b
MT		 1415+ // Remove all content
1416+ loc_network_list_clear(list);
05db64d0
MT		 1417+
1418+ loc_unref(list->ctx);
1419+ free(list);
1420+}
1421+
b952a52b 1422+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
05db64d0
MT		 1423+ if (!list)
1424+ return NULL;
1425+
1426+ if (--list->refcount > 0)
1427+ return list;
1428+
b952a52b 1429+ loc_network_list_free(list);
05db64d0
MT		 1430+ return NULL;
1431+}
1432+
b952a52b 1433+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
05db64d0
MT		 1434+ return list->size;
1435+}
1436+
b952a52b 1437+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
05db64d0
MT		 1438+ return list->size == 0;
1439+}
1440+
b952a52b
MT		 1441+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
1442+ if (!list->elements)
1443+ return;
05db64d0 1444+
b952a52b
MT		 1445+ for (unsigned int i = 0; i < list->size; i++)
1446+ loc_network_unref(list->elements[i]);
05db64d0 1447+
b952a52b
MT		 1448+ free(list->elements);
1449+ list->elements = NULL;
1450+ list->elements_size = 0;
05db64d0 1451+
b952a52b
MT		 1452+ list->size = 0;
1453+}
05db64d0 1454+
b952a52b
MT		 1455+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
1456+ struct loc_network* network;
1457+ char* s;
05db64d0 1458+
b952a52b
MT		 1459+ for (unsigned int i = 0; i < list->size; i++) {
1460+ network = list->elements[i];
05db64d0 1461+
b952a52b 1462+ s = loc_network_str(network);
05db64d0 1463+
b952a52b
MT		 1464+ INFO(list->ctx, "%4d: %s\n", i, s);
1465+ free(s);
1466+ }
1467+}
05db64d0 1468+
b952a52b
MT		 1469+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
1470+ // Check index
1471+ if (index >= list->size)
1472+ return NULL;
05db64d0 1473+
b952a52b
MT		 1474+ return loc_network_ref(list->elements[index]);
1475+}
05db64d0 1476+
b952a52b
MT		 1477+static off_t loc_network_list_find(struct loc_network_list* list,
1478+ struct loc_network* network, int* found) {
1479+ off_t lo = 0;
1480+ off_t hi = list->size - 1;
1481+ int result;
05db64d0 1482+
b952a52b
MT		 1483+ // Since we are working on an ordered list, there is often a good chance that
1484+ // the network we are looking for is at the end or has to go to the end.
1485+ if (hi >= 0) {
1486+ result = loc_network_cmp(network, list->elements[hi]);
05db64d0 1487+
b952a52b
MT		 1488+ // Match, so we are done
1489+ if (result == 0) {
1490+ *found = 1;
05db64d0 1491+
b952a52b
MT		 1492+ return hi;
1493+
1494+ // This needs to be added after the last one
1495+ } else if (result > 0) {
1496+ *found = 0;
1497+
1498+ return hi + 1;
1499+ }
1500+ }
05db64d0 1501+
b952a52b
MT		 1502+#ifdef ENABLE_DEBUG
1503+ // Save start time
1504+ clock_t start = clock();
05db64d0 1505+#endif
05db64d0 1506+
b952a52b 1507+ off_t i = 0;
05db64d0 1508+
b952a52b
MT		 1509+ while (lo <= hi) {
1510+ i = (lo + hi) / 2;
05db64d0 1511+
b952a52b
MT		 1512+ // Check if this is a match
1513+ result = loc_network_cmp(network, list->elements[i]);
05db64d0 1514+
b952a52b
MT		 1515+ if (result == 0) {
1516+ *found = 1;
05db64d0 1517+
b952a52b
MT		 1518+#ifdef ENABLE_DEBUG
1519+ clock_t end = clock();
05db64d0 1520+
b952a52b
MT		 1521+ // Log how fast this has been
1522+ DEBUG(list->ctx, "Found network in %.4fms at %jd\n",
1523+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
1524+#endif
05db64d0 1525+
b952a52b
MT		 1526+ return i;
1527+ }
05db64d0 1528+
b952a52b
MT		 1529+ if (result > 0) {
1530+ lo = i + 1;
1531+ i++;
1532+ } else {
1533+ hi = i - 1;
05db64d0 1534+ }
b952a52b 1535+ }
05db64d0 1536+
b952a52b 1537+ *found = 0;
05db64d0 1538+
b952a52b
MT		 1539+#ifdef ENABLE_DEBUG
1540+ clock_t end = clock();
05db64d0 1541+
b952a52b
MT		 1542+ // Log how fast this has been
1543+ DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n",
1544+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
1545+#endif
05db64d0 1546+
b952a52b
MT		 1547+ return i;
1548+}
1549+
1550+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
1551+ int found = 0;
1552+
1553+ off_t index = loc_network_list_find(list, network, &found);
1554+
1555+ // The network has been found on the list. Nothing to do.
1556+ if (found)
1557+ return 0;
1558+
1559+ DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n",
1560+ list, network, (intmax_t)index);
05db64d0 1561+
b952a52b
MT		 1562+ // Check if we have space left
1563+ if (list->size >= list->elements_size) {
1564+ int r = loc_network_list_grow(list, 64);
1565+ if (r)
1566+ return r;
1567+ }
05db64d0 1568+
b952a52b
MT		 1569+ // The list is now larger
1570+ list->size++;
05db64d0 1571+
b952a52b
MT		 1572+ // Move all elements out of the way
1573+ for (unsigned int i = list->size - 1; i > index; i--)
1574+ list->elements[i] = list->elements[i - 1];
05db64d0 1575+
b952a52b
MT		 1576+ // Add the new element at the right place
1577+ list->elements[index] = loc_network_ref(network);
05db64d0 1578+
b952a52b
MT		 1579+ return 0;
1580+}
05db64d0 1581+
b952a52b
MT		 1582+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
1583+ // Return nothing when empty
1584+ if (loc_network_list_empty(list)) {
1585+ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
1586+ return NULL;
1587+ }
05db64d0 1588+
b952a52b 1589+ struct loc_network* network = list->elements[--list->size];
05db64d0 1590+
b952a52b 1591+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
05db64d0 1592+
b952a52b
MT		 1593+ return network;
1594+}
1595+
1596+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
1597+ // Return nothing when empty
1598+ if (loc_network_list_empty(list)) {
1599+ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
1600+ return NULL;
1601+ }
05db64d0 1602+
b952a52b 1603+ struct loc_network* network = list->elements[0];
05db64d0 1604+
b952a52b
MT		 1605+ // Move all elements to the top of the stack
1606+ for (unsigned int i = 0; i < list->size - 1; i++) {
1607+ list->elements[i] = list->elements[i+1];
1608+ }
1609+
1610+ // The list is shorter now
1611+ --list->size;
1612+
1613+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
1614+
1615+ return network;
1616+}
1617+
1618+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
1619+ int found = 0;
1620+
1621+ loc_network_list_find(list, network, &found);
1622+
1623+ return found;
1624+}
1625+
1626+LOC_EXPORT int loc_network_list_merge(
1627+ struct loc_network_list* self, struct loc_network_list* other) {
1628+ int r;
1629+
1630+ for (unsigned int i = 0; i < other->size; i++) {
1631+ r = loc_network_list_push(self, other->elements[i]);
1632+ if (r)
1633+ return r;
1634+ }
05db64d0
MT		 1635+
1636+ return 0;
1637+}
b952a52b
MT		 1638diff --git a/src/network.c b/src/network.c
1639index 366caa2..a6b679c 100644
1640--- a/src/network.c
1641+++ b/src/network.c
1642@@ -29,6 +29,7 @@
1643 #include <loc/compat.h>
1644 #include <loc/country.h>
1645 #include <loc/network.h>
1646+#include <loc/network-list.h>
1647 #include <loc/private.h>
1648
1649 struct loc_network {
1650@@ -97,6 +98,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
1651 return a;
1652 }
1653
1654+static struct in6_addr address_increment(const struct in6_addr* address) {
1655+ struct in6_addr a = *address;
05db64d0 1656+
b952a52b
MT		 1657+ for (int octet = 15; octet >= 0; octet--) {
1658+ if (a.s6_addr[octet] < 255) {
1659+ a.s6_addr[octet]++;
1660+ break;
1661+ } else {
1662+ a.s6_addr[octet] = 0;
1663+ }
1664+ }
05db64d0 1665+
b952a52b 1666+ return a;
05db64d0
MT		 1667+}
1668+
b952a52b
MT		 1669 LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
1670 struct in6_addr* address, unsigned int prefix) {
1671 // Address cannot be unspecified
1672@@ -160,9 +176,11 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
1673 LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
1674 const char* address_string) {
1675 struct in6_addr first_address;
1676- unsigned int prefix = 0;
1677 char* prefix_string;
1678- int r = 1;
1679+ unsigned int prefix = 128;
1680+ int r = -EINVAL;
05db64d0 1681+
b952a52b
MT		 1682+ DEBUG(ctx, "Attempting to parse network %s\n", address_string);
1683
1684 // Make a copy of the string to work on it
1685 char* buffer = strdup(address_string);
1686@@ -171,29 +189,40 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1687 // Split address and prefix
1688 address_string = strsep(&prefix_string, "/");
1689
1690- // Did we find a prefix?
1691+ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
05db64d0 1692+
b952a52b
MT		 1693+ // Parse the address
1694+ r = loc_parse_address(ctx, address_string, &first_address);
1695+ if (r) {
1696+ DEBUG(ctx, "The address could not be parsed\n");
1697+ goto FAIL;
1698+ }
1699+
1700+ // If a prefix was given, we will try to parse it
1701 if (prefix_string) {
1702 // Convert prefix to integer
1703 prefix = strtol(prefix_string, NULL, 10);
1704
1705- if (prefix) {
1706- // Parse the address
1707- r = loc_parse_address(ctx, address_string, &first_address);
1708-
1709- // Map the prefix to IPv6 if needed
1710- if (IN6_IS_ADDR_V4MAPPED(&first_address))
1711- prefix += 96;
1712+ if (!prefix) {
1713+ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
1714+ goto FAIL;
1715 }
1716+
1717+ // Map the prefix to IPv6 if needed
1718+ if (IN6_IS_ADDR_V4MAPPED(&first_address))
1719+ prefix += 96;
1720 }
1721
1722+FAIL:
1723 // Free temporary buffer
1724 free(buffer);
1725
1726- if (r == 0) {
1727- r = loc_network_new(ctx, network, &first_address, prefix);
1728- }
1729+ // Exit if the parsing was unsuccessful
1730+ if (r)
1731+ return r;
1732
1733- return r;
1734+ // Create a new network
1735+ return loc_network_new(ctx, network, &first_address, prefix);
1736 }
1737
1738 LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
1739@@ -281,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) {
1740 return network->family;
1741 }
1742
1743+LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) {
1744+ switch (network->family) {
1745+ case AF_INET6:
1746+ return network->prefix;
1747+
1748+ case AF_INET:
1749+ return network->prefix - 96;
1750+ }
1751+
1752+ return 0;
05db64d0
MT		 1753+}
1754+
b952a52b
MT		 1755 static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) {
1756 const size_t length = INET6_ADDRSTRLEN;
1757
1758@@ -314,10 +355,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc
1759 return string;
1760 }
1761
1762+LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) {
1763+ return &network->first_address;
1764+}
05db64d0 1765+
b952a52b
MT		 1766 LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) {
1767 return loc_network_format_address(network, &network->first_address);
1768 }
1769
1770+LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) {
1771+ return &network->last_address;
1772+}
1773+
1774 LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
1775 return loc_network_format_address(network, &network->last_address);
1776 }
1777@@ -325,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
1778 LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) {
1779 // Address must be larger than the start address
1780 if (in6_addr_cmp(&network->first_address, address) > 0)
1781- return 1;
1782+ return 0;
1783
1784 // Address must be smaller than the last address
1785 if (in6_addr_cmp(&network->last_address, address) < 0)
1786- return 1;
1787+ return 0;
1788
1789 // The address is inside this network
1790- return 0;
1791+ return 1;
1792 }
1793
1794 LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) {
1795@@ -392,20 +441,310 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
1796 return loc_network_has_flag(network, flag);
1797 }
1798
1799-LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
1800+LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
1801+ // Compare address
1802+ int r = in6_addr_cmp(&self->first_address, &other->first_address);
1803+ if (r)
1804+ return r;
1805+
1806+ // Compare prefix
1807+ if (self->prefix > other->prefix)
1808+ return 1;
1809+ else if (self->prefix < other->prefix)
1810+ return -1;
05db64d0 1811+
b952a52b
MT		 1812+ // Both networks are equal
1813+ return 0;
05db64d0
MT		 1814+}
1815+
b952a52b
MT		 1816+LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
1817+ // Either of the start addresses must be in the other subnet
1818+ if (loc_network_match_address(self, &other->first_address))
1819+ return 1;
05db64d0 1820+
b952a52b
MT		 1821+ if (loc_network_match_address(other, &self->first_address))
1822+ return 1;
05db64d0 1823+
b952a52b
MT		 1824+ // Or either of the end addresses is in the other subnet
1825+ if (loc_network_match_address(self, &other->last_address))
1826+ return 1;
05db64d0 1827+
b952a52b
MT		 1828+ if (loc_network_match_address(other, &self->last_address))
1829+ return 1;
05db64d0 1830+
b952a52b 1831+ return 0;
05db64d0
MT		 1832+}
1833+
b952a52b
MT		 1834+LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
1835+ // The prefix must be smaller (this avoids the more complex comparisons later)
1836+ if (self->prefix > other->prefix)
05db64d0
MT		 1837+ return 0;
1838+
b952a52b
MT		 1839 // If the start address of the other network is smaller than this network,
1840 // it cannot be a subnet.
1841- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
1842+ if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
1843 return 0;
1844
1845 // If the end address of the other network is greater than this network,
1846 // it cannot be a subnet.
1847- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
1848+ if (in6_addr_cmp(&self->last_address, &other->last_address) < 0)
1849 return 0;
1850
1851 return 1;
1852 }
1853
1854+LOC_EXPORT int loc_network_subnets(struct loc_network* network,
1855+ struct loc_network** subnet1, struct loc_network** subnet2) {
1856+ int r;
1857+ *subnet1 = NULL;
1858+ *subnet2 = NULL;
05db64d0 1859+
b952a52b
MT		 1860+ // New prefix length
1861+ unsigned int prefix = network->prefix + 1;
1862+
1863+ // Check if the new prefix is valid
1864+ if (valid_prefix(&network->first_address, prefix))
1865+ return -1;
05db64d0 1866+
b952a52b
MT		 1867+ // Create the first half of the network
1868+ r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix);
1869+ if (r)
1870+ return r;
05db64d0 1871+
b952a52b
MT		 1872+ // The next subnet starts after the first one
1873+ struct in6_addr first_address = address_increment(&(*subnet1)->last_address);
05db64d0 1874+
b952a52b
MT		 1875+ // Create the second half of the network
1876+ r = loc_network_new(network->ctx, subnet2, &first_address, prefix);
1877+ if (r)
1878+ return r;
1879+
1880+ // Copy country code
1881+ const char* country_code = loc_network_get_country_code(network);
1882+ if (country_code) {
1883+ loc_network_set_country_code(*subnet1, country_code);
1884+ loc_network_set_country_code(*subnet2, country_code);
1885+ }
1886+
1887+ // Copy ASN
1888+ uint32_t asn = loc_network_get_asn(network);
1889+ if (asn) {
1890+ loc_network_set_asn(*subnet1, asn);
1891+ loc_network_set_asn(*subnet2, asn);
05db64d0
MT		 1892+ }
1893+
b952a52b
MT		 1894+ // Copy flags
1895+ loc_network_set_flag(*subnet1, network->flags);
1896+ loc_network_set_flag(*subnet2, network->flags);
1897+
05db64d0
MT		 1898+ return 0;
1899+}
1900+
b952a52b
MT		 1901+static int __loc_network_exclude(struct loc_network* network,
1902+ struct loc_network* other, struct loc_network_list* list) {
1903+ struct loc_network* subnet1 = NULL;
1904+ struct loc_network* subnet2 = NULL;
05db64d0 1905+
b952a52b 1906+ int r = loc_network_subnets(network, &subnet1, &subnet2);
05db64d0 1907+ if (r)
b952a52b 1908+ goto ERROR;
05db64d0 1909+
b952a52b
MT		 1910+ if (loc_network_cmp(other, subnet1) == 0) {
1911+ r = loc_network_list_push(list, subnet2);
1912+ if (r)
1913+ goto ERROR;
05db64d0 1914+
b952a52b
MT		 1915+ } else if (loc_network_cmp(other, subnet2) == 0) {
1916+ r = loc_network_list_push(list, subnet1);
1917+ if (r)
1918+ goto ERROR;
05db64d0 1919+
b952a52b
MT		 1920+ } else if (loc_network_is_subnet(subnet1, other)) {
1921+ r = loc_network_list_push(list, subnet2);
1922+ if (r)
1923+ goto ERROR;
05db64d0 1924+
b952a52b
MT		 1925+ r = __loc_network_exclude(subnet1, other, list);
1926+ if (r)
1927+ goto ERROR;
05db64d0 1928+
b952a52b
MT		 1929+ } else if (loc_network_is_subnet(subnet2, other)) {
1930+ r = loc_network_list_push(list, subnet1);
1931+ if (r)
1932+ goto ERROR;
05db64d0 1933+
b952a52b
MT		 1934+ r = __loc_network_exclude(subnet2, other, list);
1935+ if (r)
1936+ goto ERROR;
05db64d0 1937+
b952a52b
MT		 1938+ } else {
1939+ ERROR(network->ctx, "We should never get here\n");
1940+ r = 1;
1941+ goto ERROR;
1942+ }
05db64d0 1943+
b952a52b
MT		 1944+ERROR:
1945+ if (subnet1)
1946+ loc_network_unref(subnet1);
05db64d0 1947+
b952a52b
MT		 1948+ if (subnet2)
1949+ loc_network_unref(subnet2);
05db64d0 1950+
b952a52b
MT		 1951+ return r;
1952+}
05db64d0 1953+
b952a52b
MT		 1954+static int __loc_network_exclude_to_list(struct loc_network* self,
1955+ struct loc_network* other, struct loc_network_list* list) {
1956+ // Other must be a subnet of self
1957+ if (!loc_network_is_subnet(self, other)) {
1958+ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
1959+
1960+ // Exit silently
1961+ return 0;
1962+ }
1963+
1964+ // We cannot perform this operation if both networks equal
1965+ if (loc_network_cmp(self, other) == 0) {
1966+ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
1967+
1968+ // Exit silently
1969+ return 0;
1970+ }
05db64d0 1971+
b952a52b
MT		 1972+ return __loc_network_exclude(self, other, list);
1973+}
05db64d0 1974+
b952a52b
MT		 1975+LOC_EXPORT struct loc_network_list* loc_network_exclude(
1976+ struct loc_network* self, struct loc_network* other) {
1977+ struct loc_network_list* list;
05db64d0 1978+
b952a52b
MT		 1979+#ifdef ENABLE_DEBUG
1980+ char* n1 = loc_network_str(self);
1981+ char* n2 = loc_network_str(other);
05db64d0 1982+
b952a52b 1983+ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
05db64d0 1984+
b952a52b
MT		 1985+ free(n1);
1986+ free(n2);
1987+#endif
05db64d0 1988+
b952a52b
MT		 1989+ // Create a new list with the result
1990+ int r = loc_network_list_new(self->ctx, &list);
1991+ if (r) {
1992+ ERROR(self->ctx, "Could not create network list: %d\n", r);
05db64d0 1993+
b952a52b
MT		 1994+ return NULL;
1995+ }
05db64d0 1996+
b952a52b
MT		 1997+ r = __loc_network_exclude_to_list(self, other, list);
1998+ if (r) {
1999+ loc_network_list_unref(list);
05db64d0 2000+
b952a52b
MT		 2001+ return NULL;
2002+ }
05db64d0 2003+
b952a52b
MT		 2004+ // Return the result
2005+ return list;
2006+}
05db64d0 2007+
b952a52b
MT		 2008+LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
2009+ struct loc_network* network, struct loc_network_list* list) {
2010+ struct loc_network_list* to_check;
05db64d0 2011+
b952a52b
MT		 2012+ // Create a new list with all networks to look at
2013+ int r = loc_network_list_new(network->ctx, &to_check);
2014+ if (r)
2015+ return NULL;
05db64d0 2016+
b952a52b
MT		 2017+ struct loc_network* subnet = NULL;
2018+ struct loc_network_list* subnets = NULL;
05db64d0 2019+
b952a52b
MT		 2020+ for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
2021+ subnet = loc_network_list_get(list, i);
2022+
2023+ // Find all excluded networks
2024+ if (!loc_network_list_contains(to_check, subnet)) {
2025+ r = __loc_network_exclude_to_list(network, subnet, to_check);
05db64d0 2026+ if (r) {
b952a52b
MT		 2027+ loc_network_list_unref(to_check);
2028+ loc_network_unref(subnet);
05db64d0 2029+
05db64d0
MT		 2030+ return NULL;
2031+ }
05db64d0
MT		 2032+ }
2033+
b952a52b
MT		 2034+ // Cleanup
2035+ loc_network_unref(subnet);
05db64d0 2036+ }
05db64d0 2037+
b952a52b
MT		 2038+ r = loc_network_list_new(network->ctx, &subnets);
2039+ if (r) {
2040+ loc_network_list_unref(to_check);
2041+ return NULL;
05db64d0 2042+ }
05db64d0 2043+
b952a52b 2044+ off_t smallest_subnet = 0;
05db64d0 2045+
b952a52b
MT		 2046+ while (!loc_network_list_empty(to_check)) {
2047+ struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check);
05db64d0 2048+
b952a52b
MT		 2049+ // Check whether the subnet to check is part of the input list
2050+ if (loc_network_list_contains(list, subnet_to_check)) {
2051+ loc_network_unref(subnet_to_check);
2052+ continue;
2053+ }
05db64d0 2054+
b952a52b
MT		 2055+ // Marks whether this subnet passed all checks
2056+ int passed = 1;
05db64d0 2057+
b952a52b
MT		 2058+ for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) {
2059+ subnet = loc_network_list_get(list, i);
2060+
2061+ // Drop this subnet if is a subnet of another subnet
2062+ if (loc_network_is_subnet(subnet, subnet_to_check)) {
2063+ passed = 0;
2064+ loc_network_unref(subnet);
2065+ break;
2066+ }
05db64d0 2067+
b952a52b
MT		 2068+ // Break it down if it overlaps
2069+ if (loc_network_overlaps(subnet, subnet_to_check)) {
2070+ passed = 0;
05db64d0 2071+
b952a52b 2072+ __loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
05db64d0 2073+
b952a52b
MT		 2074+ loc_network_unref(subnet);
2075+ break;
2076+ }
05db64d0 2077+
b952a52b
MT		 2078+ // If the subnet is strictly greater, we do not need to continue the search
2079+ r = loc_network_cmp(subnet, subnet_to_check);
2080+ if (r > 0) {
2081+ loc_network_unref(subnet);
2082+ break;
05db64d0 2083+
b952a52b
MT		 2084+ // If it is strictly smaller, we can continue the search from here next
2085+ // time because all networks that are to be checked can only be larger
2086+ // than this one.
2087+ } else if (r < 0) {
2088+ smallest_subnet = i;
2089+ }
05db64d0 2090+
b952a52b
MT		 2091+ loc_network_unref(subnet);
2092+ }
05db64d0 2093+
b952a52b
MT		 2094+ if (passed) {
2095+ r = loc_network_list_push(subnets, subnet_to_check);
2096+ }
05db64d0 2097+
b952a52b
MT		 2098+ loc_network_unref(subnet_to_check);
2099+ }
05db64d0 2100+
b952a52b
MT		 2101+ loc_network_list_unref(to_check);
2102+
2103+ return subnets;
05db64d0
MT		 2104+}
2105+
b952a52b
MT		 2106 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
2107 // Add country code
2108 loc_country_code_copy(dbobj->country_code, network->country_code);
2109@@ -474,7 +813,7 @@ struct loc_network_tree_node {
2110 struct loc_network* network;
2111 };
05db64d0 2112
b952a52b
MT		 2113-LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
2114+int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
2115 struct loc_network_tree* t = calloc(1, sizeof(*t));
2116 if (!t)
2117 return -ENOMEM;
2118@@ -494,7 +833,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
05db64d0
MT		 2119 return 0;
2120 }
05db64d0 2121
b952a52b
MT		 2122-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
2123+struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
2124 return loc_network_tree_node_ref(tree->root);
05db64d0
MT		 2125 }
2126
b952a52b
MT		 2127@@ -566,7 +905,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
2128 return 0;
05db64d0
MT		 2129 }
2130
b952a52b
MT		 2131-LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
2132+int loc_network_tree_walk(struct loc_network_tree* tree,
2133 int(*filter_callback)(struct loc_network* network, void* data),
2134 int(*callback)(struct loc_network* network, void* data), void* data) {
2135 return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
2136@@ -581,7 +920,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
2137 free(tree);
05db64d0
MT		 2138 }
2139
b952a52b
MT		 2140-LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
2141+struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
2142 if (--tree->refcount > 0)
2143 return tree;
05db64d0 2144
b952a52b
MT		 2145@@ -602,13 +941,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
2146 return 0;
05db64d0
MT		 2147 }
2148
b952a52b
MT		 2149-LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
2150+int loc_network_tree_dump(struct loc_network_tree* tree) {
2151 DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
05db64d0 2152
b952a52b
MT		 2153 return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
2154 }
2155
2156-LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
2157+int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
2158 DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
05db64d0 2159
b952a52b
MT		 2160 struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
2161@@ -639,7 +978,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
05db64d0 2162 return 0;
b952a52b 2163 }
05db64d0 2164
b952a52b
MT		 2165-LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
2166+size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
2167 size_t counter = 0;
78a6918d 2168
b952a52b
MT		 2169 int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
2170@@ -661,11 +1000,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
2171 return counter;
2172 }
78a6918d 2173
b952a52b
MT		 2174-LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
2175+size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
2176 return __loc_network_tree_count_nodes(tree->root);
2177 }
78a6918d 2178
b952a52b
MT		 2179-LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
2180+int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
2181 struct loc_network_tree_node* n = calloc(1, sizeof(*n));
2182 if (!n)
2183 return -ENOMEM;
2184@@ -680,7 +1019,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
2185 return 0;
2186 }
78a6918d 2187
b952a52b
MT		 2188-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
2189+struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
2190 if (node)
2191 node->refcount++;
78a6918d 2192
b952a52b
MT		 2193@@ -703,7 +1042,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
2194 free(node);
2195 }
78a6918d 2196
b952a52b
MT		 2197-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
2198+struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
2199 if (!node)
2200 return NULL;
78a6918d 2201
b952a52b
MT		 2202@@ -714,7 +1053,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
2203 return NULL;
2204 }
78a6918d 2205
b952a52b
MT		 2206-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
2207+struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
2208 if (index == 0)
2209 node = node->zero;
2210 else
2211@@ -726,10 +1065,10 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
2212 return loc_network_tree_node_ref(node);
2213 }
78a6918d 2214
b952a52b
MT		 2215-LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
2216+int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
2217 return (!!node->network);
2218 }
78a6918d 2219
b952a52b
MT		 2220-LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2221+struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2222 return loc_network_ref(node->network);
2223 }
2224diff --git a/src/perl/Location.xs b/src/perl/Location.xs
2225index dcf3f0d..b7676d2 100644
2226--- a/src/perl/Location.xs
2227+++ b/src/perl/Location.xs
2228@@ -125,7 +125,7 @@ database_countries(db)
2229 PPCODE:
2230 // Create Database enumerator
2231 struct loc_database_enumerator* enumerator;
2232- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
2233+ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
78a6918d 2234
b952a52b
MT		 2235 if (err) {
2236 croak("Could not create a database enumerator\n");
2237diff --git a/src/python/database.c b/src/python/database.c
2238index 1013a58..f385c61 100644
2239--- a/src/python/database.c
2240+++ b/src/python/database.c
2241@@ -17,6 +17,8 @@
2242 #include <Python.h>
78a6918d 2243
b952a52b
MT		 2244 #include <loc/libloc.h>
2245+#include <loc/as.h>
2246+#include <loc/as-list.h>
2247 #include <loc/database.h>
78a6918d 2248
b952a52b
MT		 2249 #include "locationmodule.h"
2250@@ -207,10 +209,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
2251 return (PyObject*)self;
2252 }
78a6918d 2253
b952a52b
MT		 2254-static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
2255+static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
2256 struct loc_database_enumerator* enumerator;
78a6918d 2257
b952a52b
MT		 2258- int r = loc_database_enumerator_new(&enumerator, self->db, what);
2259+ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
2260 if (r) {
2261 PyErr_SetFromErrno(PyExc_SystemError);
2262 return NULL;
2263@@ -223,7 +225,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
2264 }
78a6918d 2265
b952a52b
MT		 2266 static PyObject* Database_ases(DatabaseObject* self) {
2267- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
2268+ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
2269 }
78a6918d 2270
b952a52b
MT		 2271 static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2272@@ -234,7 +236,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2273
2274 struct loc_database_enumerator* enumerator;
2275
2276- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
2277+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
2278 if (r) {
2279 PyErr_SetFromErrno(PyExc_SystemError);
2280 return NULL;
2281@@ -250,44 +252,142 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
78a6918d
MT		 2282 }
2283
b952a52b
MT		 2284 static PyObject* Database_networks(DatabaseObject* self) {
2285- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
2286+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
78a6918d
MT		 2287+}
2288+
b952a52b
MT		 2289+static PyObject* Database_networks_flattened(DatabaseObject *self) {
2290+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
2291 }
2292
2293 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
2294- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
2295- const char* country_code = NULL;
2296- unsigned int asn = 0;
2297+ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
2298+ PyObject* country_codes = NULL;
2299+ PyObject* asn_list = NULL;
2300 int flags = 0;
2301 int family = 0;
2302+ int flatten = 0;
2303
2304- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
2305+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
2306+ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
2307 return NULL;
2308
2309 struct loc_database_enumerator* enumerator;
2310- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
2311+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
2312+ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
2313 if (r) {
2314 PyErr_SetFromErrno(PyExc_SystemError);
2315 return NULL;
2316 }
2317
2318 // Set country code we are searching for
2319- if (country_code) {
2320- r = loc_database_enumerator_set_country_code(enumerator, country_code);
2321+ if (country_codes) {
2322+ struct loc_country_list* countries;
2323+ r = loc_country_list_new(loc_ctx, &countries);
2324+ if (r) {
2325+ PyErr_SetString(PyExc_SystemError, "Could not create country list");
2326+ return NULL;
2327+ }
2328+
2329+ for (unsigned int i = 0; i < PyList_Size(country_codes); i++) {
2330+ PyObject* item = PyList_GetItem(country_codes, i);
2331+
2332+ if (!PyUnicode_Check(item)) {
2333+ PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
2334+ loc_country_list_unref(countries);
2335+ return NULL;
2336+ }
2337+
2338+ const char* country_code = PyUnicode_AsUTF8(item);
2339+
2340+ struct loc_country* country;
2341+ r = loc_country_new(loc_ctx, &country, country_code);
2342+ if (r) {
2343+ if (r == -EINVAL) {
2344+ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
2345+ } else {
2346+ PyErr_SetString(PyExc_SystemError, "Could not create country");
2347+ }
2348+
2349+ loc_country_list_unref(countries);
2350+ return NULL;
2351+ }
2352+
2353+ // Append it to the list
2354+ r = loc_country_list_append(countries, country);
2355+ if (r) {
2356+ PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
78a6918d 2357+
b952a52b
MT		 2358+ loc_country_list_unref(countries);
2359+ loc_country_unref(country);
2360+ return NULL;
2361+ }
2362+
2363+ loc_country_unref(country);
2364+ }
2365
2366+ r = loc_database_enumerator_set_countries(enumerator, countries);
78a6918d
MT		 2367 if (r) {
2368 PyErr_SetFromErrno(PyExc_SystemError);
b952a52b 2369+
78a6918d
MT		 2370+ loc_country_list_unref(countries);
2371 return NULL;
2372 }
b952a52b
MT		 2373+
2374+ loc_country_list_unref(countries);
2375 }
78a6918d 2376
b952a52b
MT		 2377 // Set the ASN we are searching for
2378- if (asn) {
2379- r = loc_database_enumerator_set_asn(enumerator, asn);
2380+ if (asn_list) {
2381+ struct loc_as_list* asns;
2382+ r = loc_as_list_new(loc_ctx, &asns);
2383+ if (r) {
2384+ PyErr_SetString(PyExc_SystemError, "Could not create AS list");
2385+ return NULL;
2386+ }
2387+
2388+ for (unsigned int i = 0; i < PyList_Size(asn_list); i++) {
2389+ PyObject* item = PyList_GetItem(asn_list, i);
2390+
2391+ if (!PyLong_Check(item)) {
2392+ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
78a6918d 2393
b952a52b
MT		 2394+ loc_as_list_unref(asns);
2395+ return NULL;
2396+ }
78a6918d 2397+
b952a52b 2398+ unsigned long number = PyLong_AsLong(item);
78a6918d 2399+
b952a52b
MT		 2400+ struct loc_as* as;
2401+ r = loc_as_new(loc_ctx, &as, number);
2402+ if (r) {
2403+ PyErr_SetString(PyExc_SystemError, "Could not create AS");
78a6918d 2404+
b952a52b
MT		 2405+ loc_as_list_unref(asns);
2406+ loc_as_unref(as);
2407+ return NULL;
2408+ }
78a6918d 2409+
b952a52b
MT		 2410+ r = loc_as_list_append(asns, as);
2411+ if (r) {
2412+ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
78a6918d 2413+
b952a52b
MT		 2414+ loc_as_list_unref(asns);
2415+ loc_as_unref(as);
2416+ return NULL;
2417+ }
78a6918d 2418+
b952a52b
MT		 2419+ loc_as_unref(as);
2420+ }
2421+
2422+ r = loc_database_enumerator_set_asns(enumerator, asns);
2423 if (r) {
2424 PyErr_SetFromErrno(PyExc_SystemError);
2425+
2426+ loc_as_list_unref(asns);
2427 return NULL;
2428 }
2429+
2430+ loc_as_list_unref(asns);
2431 }
78a6918d 2432
b952a52b
MT		 2433 // Set the flags we are searching for
2434@@ -317,7 +417,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
78a6918d
MT		 2435 }
2436
b952a52b
MT		 2437 static PyObject* Database_countries(DatabaseObject* self) {
2438- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
2439+ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
2440 }
2441
2442 static struct PyMethodDef Database_methods[] = {
2443@@ -403,6 +503,13 @@ static struct PyGetSetDef Database_getsetters[] = {
78a6918d
MT		 2444 NULL,
2445 NULL,
2446 },
2447+ {
b952a52b
MT		 2448+ "networks_flattened",
2449+ (getter)Database_networks_flattened,
78a6918d
MT		 2450+ NULL,
2451+ NULL,
2452+ NULL,
2453+ },
2454 {
b952a52b
MT		 2455 "vendor",
2456 (getter)Database_get_vendor,
2457diff --git a/src/python/downloader.py b/src/python/downloader.py
2458index 87bbb68..05f7872 100644
2459--- a/src/python/downloader.py
2460+++ b/src/python/downloader.py
2461@@ -119,8 +119,8 @@ class Downloader(object):
2462
2463 headers = {}
2464 if timestamp:
2465- headers["If-Modified-Since"] = timestamp.strftime(
2466- "%a, %d %b %Y %H:%M:%S GMT",
2467+ headers["If-Modified-Since"] = time.strftime(
2468+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
2469 )
2470
2471 t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
2472@@ -195,7 +195,7 @@ class Downloader(object):
2473 db = Database(f.name)
2474
2475 # Database is not recent
2476- if timestamp and db.created_at < timestamp.timestamp():
2477+ if timestamp and db.created_at < timestamp:
2478 return False
78a6918d 2479
b952a52b 2480 log.info("Downloaded new database from %s" % (time.strftime(
78a6918d 2481diff --git a/src/python/export.py b/src/python/export.py
b952a52b 2482index d15c6f0..f0eae26 100644
78a6918d
MT		 2483--- a/src/python/export.py
2484+++ b/src/python/export.py
b952a52b
MT		 2485@@ -29,7 +29,7 @@ import _location
2486 log = logging.getLogger("location.export")
2487 log.propagate = 1
2488
2489-flags = {
2490+FLAGS = {
2491 _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
2492 _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
2493 _location.NETWORK_FLAG_ANYCAST : "A3",
78a6918d
MT		 2494@@ -39,11 +39,8 @@ class OutputWriter(object):
2495 suffix = "networks"
2496 mode = "w"
2497
2498- def __init__(self, f, prefix=None, flatten=True):
2499- self.f, self.prefix, self.flatten = f, prefix, flatten
2500-
2501- # The previously written network
2502- self._last_network = None
2503+ def __init__(self, f, prefix=None):
2504+ self.f, self.prefix = f, prefix
2505
2506 # Immediately write the header
2507 self._write_header()
2508@@ -60,18 +57,6 @@ class OutputWriter(object):
2509 def __repr__(self):
2510 return "<%s f=%s>" % (self.__class__.__name__, self.f)
2511
2512- def _flatten(self, network):
2513- """
2514- Checks if the given network needs to be written to file,
2515- or if it is a subnet of the previously written network.
2516- """
2517- if self._last_network and network.is_subnet_of(self._last_network):
2518- return True
2519-
2520- # Remember this network for the next call
2521- self._last_network = network
2522- return False
2523-
2524 def _write_header(self):
2525 """
2526 The header of the file
b952a52b 2527@@ -84,16 +69,8 @@ class OutputWriter(object):
78a6918d
MT		 2528 """
2529 pass
2530
2531- def _write_network(self, network):
2532- self.f.write("%s\n" % network)
2533-
2534 def write(self, network):
2535- if self.flatten and self._flatten(network):
b952a52b 2536- log.debug("Skipping writing network %s" % network)
78a6918d
MT		 2537- return
2538-
b952a52b
MT		 2539- # Write the network to file
2540- self._write_network(network)
78a6918d
MT		 2541+ self.f.write("%s\n" % network)
2542
2543 def finish(self):
2544 """
b952a52b 2545@@ -114,7 +91,7 @@ class IpsetOutputWriter(OutputWriter):
78a6918d
MT		 2546 def _write_header(self):
2547 self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
2548
2549- def _write_network(self, network):
2550+ def write(self, network):
2551 self.f.write("add %s %s\n" % (self.prefix, network))
2552
2553
b952a52b 2554@@ -130,7 +107,7 @@ class NftablesOutputWriter(OutputWriter):
78a6918d
MT		 2555 def _write_footer(self):
2556 self.f.write("}\n")
2557
2558- def _write_network(self, network):
2559+ def write(self, network):
b952a52b 2560 self.f.write(" %s,\n" % network)
78a6918d 2561
78a6918d 2562
b952a52b
MT		 2563@@ -142,14 +119,9 @@ class XTGeoIPOutputWriter(OutputWriter):
2564 suffix = "iv"
2565 mode = "wb"
78a6918d 2566
b952a52b
MT		 2567- def _write_network(self, network):
2568- for address in (network.first_address, network.last_address):
2569- # Convert this into a string of bits
2570- bytes = socket.inet_pton(
2571- network.family, address,
2572- )
2573-
2574- self.f.write(bytes)
2575+ def write(self, network):
2576+ self.f.write(network._first_address)
2577+ self.f.write(network._last_address)
78a6918d 2578
78a6918d 2579
b952a52b
MT		 2580 formats = {
2581@@ -185,8 +157,14 @@ class Exporter(object):
78a6918d 2582
b952a52b 2583 writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
78a6918d 2584
b952a52b
MT		 2585+ # Filter countries from special country codes
2586+ country_codes = [
2587+ country_code for country_code in countries if not country_code in FLAGS.values()
2588+ ]
2589+
2590 # Get all networks that match the family
2591- networks = self.db.search_networks(family=family)
2592+ networks = self.db.search_networks(family=family,
2593+ country_codes=country_codes, asns=asns, flatten=True)
78a6918d 2594
b952a52b
MT		 2595 # Walk through all networks
2596 for network in networks:
2597@@ -203,10 +181,10 @@ class Exporter(object):
2598 pass
78a6918d 2599
b952a52b
MT		 2600 # Handle flags
2601- for flag in flags:
2602+ for flag in FLAGS:
2603 if network.has_flag(flag):
2604 # Fetch the "fake" country code
2605- country = flags[flag]
2606+ country = FLAGS[flag]
0be475ca 2607
b952a52b
MT		 2608 try:
2609 writers[country].write(network)
2610diff --git a/src/python/importer.py b/src/python/importer.py
2611index f19db4b..5f46bc3 100644
2612--- a/src/python/importer.py
2613+++ b/src/python/importer.py
2614@@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
2615 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
0be475ca 2616
b952a52b
MT		 2617 # Latin America and Caribbean Network Information Centre
2618- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2619+ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
0be475ca 2620
b952a52b
MT		 2621 # Réseaux IP Européens
2622 #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
2623diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2624index 1467923..2dec89e 100644
2625--- a/src/python/location-importer.in
2626+++ b/src/python/location-importer.in
2627@@ -152,6 +152,7 @@ class CLI(object):
2628 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
2629 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
2630 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
2631+ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
2632
2633 -- autnums
2634 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
2635@@ -165,6 +166,7 @@ class CLI(object):
2636 -- networks
2637 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
2638 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
2639+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
2640 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
0be475ca 2641
b952a52b
MT		 2642 -- overrides
2643@@ -188,6 +190,8 @@ class CLI(object):
2644 );
2645 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
2646 ON network_overrides(network);
2647+ CREATE INDEX IF NOT EXISTS network_overrides_search
2648+ ON network_overrides USING GIST(network inet_ops);
2649 """)
0be475ca 2650
b952a52b
MT		 2651 return db
2652@@ -234,32 +238,24 @@ class CLI(object):
0be475ca 2653
b952a52b
MT		 2654 # Select all known networks
2655 rows = self.db.query("""
2656- -- Get a (sorted) list of all known networks
2657- WITH known_networks AS (
2658- SELECT network FROM announcements
2659- UNION
2660- SELECT network FROM networks
2661- ORDER BY network
2662- )
0be475ca 2663-
b952a52b
MT		 2664 -- Return a list of those networks enriched with all
2665 -- other information that we store in the database
2666 SELECT
2667- DISTINCT ON (known_networks.network)
2668- known_networks.network AS network,
2669- announcements.autnum AS autnum,
2670+ DISTINCT ON (network)
2671+ network,
2672+ autnum,
0be475ca 2673
b952a52b
MT		 2674 -- Country
2675 COALESCE(
2676 (
2677 SELECT country FROM network_overrides overrides
2678- WHERE announcements.network <<= overrides.network
2679+ WHERE networks.network <<= overrides.network
2680 ORDER BY masklen(overrides.network) DESC
2681 LIMIT 1
2682 ),
2683 (
2684 SELECT country FROM autnum_overrides overrides
2685- WHERE announcements.autnum = overrides.number
2686+ WHERE networks.autnum = overrides.number
2687 ),
2688 networks.country
2689 ) AS country,
2690@@ -268,50 +264,67 @@ class CLI(object):
2691 COALESCE(
2692 (
2693 SELECT is_anonymous_proxy FROM network_overrides overrides
2694- WHERE announcements.network <<= overrides.network
2695+ WHERE networks.network <<= overrides.network
2696 ORDER BY masklen(overrides.network) DESC
2697 LIMIT 1
2698 ),
2699 (
2700 SELECT is_anonymous_proxy FROM autnum_overrides overrides
2701- WHERE announcements.autnum = overrides.number
2702+ WHERE networks.autnum = overrides.number
2703 ),
2704 FALSE
2705 ) AS is_anonymous_proxy,
2706 COALESCE(
2707 (
2708 SELECT is_satellite_provider FROM network_overrides overrides
2709- WHERE announcements.network <<= overrides.network
2710+ WHERE networks.network <<= overrides.network
2711 ORDER BY masklen(overrides.network) DESC
2712 LIMIT 1
2713 ),
2714 (
2715 SELECT is_satellite_provider FROM autnum_overrides overrides
2716- WHERE announcements.autnum = overrides.number
2717+ WHERE networks.autnum = overrides.number
2718 ),
2719 FALSE
2720 ) AS is_satellite_provider,
2721 COALESCE(
2722 (
2723 SELECT is_anycast FROM network_overrides overrides
2724- WHERE announcements.network <<= overrides.network
2725+ WHERE networks.network <<= overrides.network
2726 ORDER BY masklen(overrides.network) DESC
2727 LIMIT 1
2728 ),
2729 (
2730 SELECT is_anycast FROM autnum_overrides overrides
2731- WHERE announcements.autnum = overrides.number
2732+ WHERE networks.autnum = overrides.number
2733 ),
2734 FALSE
2735- ) AS is_anycast,
0be475ca 2736-
b952a52b
MT		 2737- -- Must be part of returned values for ORDER BY clause
2738- masklen(announcements.network) AS sort_a,
2739- masklen(networks.network) AS sort_b
2740- FROM known_networks
2741- LEFT JOIN announcements ON known_networks.network <<= announcements.network
2742- LEFT JOIN networks ON known_networks.network <<= networks.network
2743- ORDER BY known_networks.network, sort_a DESC, sort_b DESC
2744+ ) AS is_anycast
2745+ FROM (
2746+ SELECT
2747+ known_networks.network AS network,
2748+ announcements.autnum AS autnum,
2749+ networks.country AS country,
0be475ca 2750+
b952a52b
MT		 2751+ -- Must be part of returned values for ORDER BY clause
2752+ masklen(announcements.network) AS sort_a,
2753+ masklen(networks.network) AS sort_b
2754+ FROM (
2755+ SELECT network FROM announcements
2756+ UNION ALL
2757+ SELECT network FROM networks
2758+ UNION ALL
2759+ SELECT network FROM network_overrides
2760+ ) known_networks
2761+ LEFT JOIN
2762+ announcements ON known_networks.network <<= announcements.network
2763+ LEFT JOIN
2764+ networks ON known_networks.network <<= networks.network
2765+ ORDER BY
2766+ known_networks.network,
2767+ sort_a DESC,
2768+ sort_b DESC
2769+ ) networks
2770 """)
0be475ca 2771
b952a52b
MT		 2772 for row in rows:
2773@@ -363,6 +376,16 @@ class CLI(object):
2774 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
2775 ON COMMIT DROP;
2776 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
0be475ca 2777+
b952a52b
MT		 2778+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
2779+ ON COMMIT DROP;
2780+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
2781+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
2782+ """)
2783+
2784+ # Remove all previously imported content
2785+ self.db.execute("""
2786+ TRUNCATE TABLE networks;
2787 """)
0be475ca 2788
b952a52b
MT		 2789 for source in location.importer.WHOIS_SOURCES:
2790@@ -370,31 +393,72 @@ class CLI(object):
2791 for block in f:
2792 self._parse_block(block)
0be475ca 2793
b952a52b
MT		 2794+ # Process all parsed networks from every RIR we happen to have access to,
2795+ # insert the largest network chunks into the networks table immediately...
2796+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
0be475ca 2797+
b952a52b
MT		 2798+ for family in (row.family for row in families):
2799+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
0be475ca 2800+
b952a52b
MT		 2801+ self.db.execute("INSERT INTO networks(network, country) \
2802+ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
0be475ca 2803+
b952a52b
MT		 2804+ # ... determine any other prefixes for this network family, ...
2805+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
2806+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
0be475ca 2807+
b952a52b
MT		 2808+ # ... and insert networks with this prefix in case they provide additional
2809+ # information (i. e. subnet of a larger chunk with a different country)
2810+ for prefix in (row.prefix for row in prefixes):
2811+ self.db.execute("""
2812+ WITH candidates AS (
2813+ SELECT
2814+ _rirdata.network,
2815+ _rirdata.country
2816+ FROM
2817+ _rirdata
2818+ WHERE
2819+ family(_rirdata.network) = %s
2820+ AND
2821+ masklen(_rirdata.network) = %s
2822+ ),
2823+ filtered AS (
2824+ SELECT
2825+ DISTINCT ON (c.network)
2826+ c.network,
2827+ c.country,
2828+ masklen(networks.network),
2829+ networks.country AS parent_country
2830+ FROM
2831+ candidates c
2832+ LEFT JOIN
2833+ networks
2834+ ON
2835+ c.network << networks.network
2836+ ORDER BY
2837+ c.network,
2838+ masklen(networks.network) DESC NULLS LAST
2839+ )
2840+ INSERT INTO
2841+ networks(network, country)
2842+ SELECT
2843+ network,
2844+ country
2845+ FROM
2846+ filtered
2847+ WHERE
2848+ parent_country IS NULL
2849+ OR
2850+ country <> parent_country
2851+ ON CONFLICT DO NOTHING""",
2852+ family, prefix,
2853+ )
0be475ca 2854+
b952a52b
MT		 2855 self.db.execute("""
2856 INSERT INTO autnums(number, name)
2857 SELECT _autnums.number, _organizations.name FROM _autnums
2858 JOIN _organizations ON _autnums.organization = _organizations.handle
2859- ON CONFLICT (number) DO UPDATE SET name = excluded.name
2860- """)
2861-
2862- self.db.execute("""
2863- --- Purge any redundant entries
2864- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
2865- AS
2866- SELECT network FROM networks candidates
2867- WHERE EXISTS (
2868- SELECT FROM networks
2869- WHERE
2870- networks.network << candidates.network
2871- AND
2872- networks.country = candidates.country
2873- );
0be475ca 2874-
b952a52b 2875- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
0be475ca 2876-
b952a52b
MT		 2877- DELETE FROM networks WHERE EXISTS (
2878- SELECT FROM _garbage WHERE networks.network = _garbage.network
2879- );
2880+ ON CONFLICT (number) DO UPDATE SET name = excluded.name;
2881 """)
0be475ca 2882
b952a52b
MT		 2883 # Download all extended sources
2884@@ -405,6 +469,69 @@ class CLI(object):
2885 for line in f:
2886 self._parse_line(line)
0be475ca 2887
b952a52b
MT		 2888+ def _check_parsed_network(self, network):
2889+ """
2890+ Assistive function to detect and subsequently sort out parsed
2891+ networks from RIR data (both Whois and so-called "extended sources"),
2892+ which are or have...
0be475ca 2893+
b952a52b
MT		 2894+ (a) not globally routable (RFC 1918 space, et al.)
2895+ (b) covering a too large chunk of the IP address space (prefix length
2896+ is < 7 for IPv4 networks, and < 10 for IPv6)
2897+ (c) "0.0.0.0" or "::" as a network address
2898+ (d) are too small for being publicly announced (we have decided not to
2899+ process them at the moment, as they significantly enlarge our
2900+ database without providing very helpful additional information)
0be475ca 2901+
b952a52b
MT		 2902+ This unfortunately is necessary due to brain-dead clutter across
2903+ various RIR databases, causing mismatches and eventually disruptions.
0be475ca 2904+
b952a52b
MT		 2905+ We will return False in case a network is not suitable for adding
2906+ it to our database, and True otherwise.
2907+ """
0be475ca 2908+
b952a52b
MT		 2909+ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
2910+ return False
0be475ca 2911+
b952a52b
MT		 2912+ if not network.is_global:
2913+ log.warning("Skipping non-globally routable network: %s" % network)
2914+ return False
0be475ca 2915+
b952a52b
MT		 2916+ if network.version == 4:
2917+ if network.prefixlen < 7:
2918+ log.warning("Skipping too big IP chunk: %s" % network)
2919+ return False
0be475ca 2920+
b952a52b
MT		 2921+ if network.prefixlen > 24:
2922+ log.debug("Skipping network too small to be publicly announced: %s" % network)
2923+ return False
0be475ca 2924+
b952a52b
MT		 2925+ if str(network.network_address) == "0.0.0.0":
2926+ log.warning("Skipping network based on 0.0.0.0: %s" % network)
2927+ return False
0be475ca 2928+
b952a52b
MT		 2929+ elif network.version == 6:
2930+ if network.prefixlen < 10:
2931+ log.warning("Skipping too big IP chunk: %s" % network)
2932+ return False
0be475ca 2933+
b952a52b
MT		 2934+ if network.prefixlen > 48:
2935+ log.debug("Skipping network too small to be publicly announced: %s" % network)
2936+ return False
0be475ca 2937+
b952a52b
MT		 2938+ if str(network.network_address) == "::":
2939+ log.warning("Skipping network based on '::': %s" % network)
2940+ return False
2941+
2942+ else:
2943+ # This should not happen...
2944+ log.warning("Skipping network of unknown family, this should not happen: %s" % network)
2945+ return False
2946+
2947+ # In case we have made it here, the network is considered to
2948+ # be suitable for libloc consumption...
2949+ return True
2950+
2951 def _parse_block(self, block):
2952 # Get first line to find out what type of block this is
2953 line = block[0]
2954@@ -433,7 +560,7 @@ class CLI(object):
2955 autnum["asn"] = m.group(2)
0be475ca 2956
b952a52b
MT		 2957 elif key == "org":
2958- autnum[key] = val
2959+ autnum[key] = val.upper()
0be475ca 2960
b952a52b
MT		 2961 # Skip empty objects
2962 if not autnum:
2963@@ -447,15 +574,22 @@ class CLI(object):
2964 )
0be475ca 2965
b952a52b
MT		 2966 def _parse_inetnum_block(self, block):
2967- logging.debug("Parsing inetnum block:")
2968+ log.debug("Parsing inetnum block:")
0be475ca 2969
b952a52b
MT		 2970 inetnum = {}
2971 for line in block:
2972- logging.debug(line)
2973+ log.debug(line)
0be475ca 2974
b952a52b
MT		 2975 # Split line
2976 key, val = split_line(line)
0be475ca 2977
b952a52b
MT		 2978+ # Filter any inetnum records which are only referring to IP space
2979+ # not managed by that specific RIR...
2980+ if key == "netname":
2981+ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
2982+ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
2983+ return
0be475ca 2984+
b952a52b
MT		 2985 if key == "inetnum":
2986 start_address, delim, end_address = val.partition("-")
2987
2988@@ -467,7 +601,7 @@ class CLI(object):
2989 start_address = ipaddress.ip_address(start_address)
2990 end_address = ipaddress.ip_address(end_address)
2991 except ValueError:
2992- logging.warning("Could not parse line: %s" % line)
2993+ log.warning("Could not parse line: %s" % line)
2994 return
2995
2996 # Set prefix to default
2997@@ -484,23 +618,24 @@ class CLI(object):
2998 inetnum[key] = val
0be475ca 2999
b952a52b
MT		 3000 elif key == "country":
3001- if val == "UNITED STATES":
3002- val = "US"
0be475ca 3003-
b952a52b 3004 inetnum[key] = val.upper()
0be475ca 3005
b952a52b
MT		 3006 # Skip empty objects
3007- if not inetnum:
3008+ if not inetnum or not "country" in inetnum:
3009+ return
3010+
3011+ # Skip objects with bogus country code 'ZZ'
3012+ if inetnum.get("country") == "ZZ":
3013+ log.warning("Skipping network with bogus country 'ZZ': %s" % \
3014+ (inetnum.get("inet6num") or inetnum.get("inetnum")))
3015 return
0be475ca 3016
b952a52b 3017 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
0be475ca 3018
b952a52b
MT		 3019- # Bail out in case we have processed a non-public IP network
3020- if network.is_private:
3021- logging.warning("Skipping non-globally routable network: %s" % network)
3022+ if not self._check_parsed_network(network):
3023 return
0be475ca 3024
b952a52b
MT		 3025- self.db.execute("INSERT INTO networks(network, country) \
3026+ self.db.execute("INSERT INTO _rirdata(network, country) \
3027 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
3028 "%s" % network, inetnum.get("country"),
3029 )
3030@@ -511,7 +646,9 @@ class CLI(object):
3031 # Split line
3032 key, val = split_line(line)
0be475ca 3033
b952a52b
MT		 3034- if key in ("organisation", "org-name"):
3035+ if key == "organisation":
3036+ org[key] = val.upper()
3037+ elif key == "org-name":
3038 org[key] = val
0be475ca 3039
b952a52b
MT		 3040 # Skip empty objects
3041@@ -581,6 +718,9 @@ class CLI(object):
3042 log.warning("Invalid IP address: %s" % address)
3043 return
0be475ca 3044
b952a52b
MT		 3045+ if not self._check_parsed_network(network):
3046+ return
0be475ca 3047+
b952a52b
MT		 3048 self.db.execute("INSERT INTO networks(network, country) \
3049 VALUES(%s, %s) ON CONFLICT (network) DO \
3050 UPDATE SET country = excluded.country",
3051diff --git a/src/python/location.in b/src/python/location.in
3052index 44ad726..b30beae 100644
3053--- a/src/python/location.in
3054+++ b/src/python/location.in
3055@@ -253,6 +253,7 @@ class CLI(object):
3056 network = db.lookup(address)
3057 except ValueError:
3058 print(_("Invalid IP address: %s") % address, file=sys.stderr)
3059+ return 2
0be475ca 3060
b952a52b
MT		 3061 args = {
3062 "address" : address,
3063@@ -398,10 +399,7 @@ class CLI(object):
0be475ca 3064
b952a52b
MT		 3065 def handle_update(self, db, ns):
3066 if ns.cron and db:
3067- now = datetime.datetime.utcnow()
0be475ca 3068-
b952a52b
MT		 3069- # Parse the database timestamp
3070- t = datetime.datetime.utcfromtimestamp(db.created_at)
3071+ now = time.time()
0be475ca 3072
b952a52b
MT		 3073 if ns.cron == "daily":
3074 delta = datetime.timedelta(days=1)
3075@@ -410,22 +408,20 @@ class CLI(object):
3076 elif ns.cron == "monthly":
3077 delta = datetime.timedelta(days=30)
0be475ca 3078
b952a52b
MT		 3079+ delta = delta.total_seconds()
3080+
3081 # Check if the database has recently been updated
3082- if t >= (now - delta):
3083+ if db.created_at >= (now - delta):
3084 log.info(
3085- _("The database has been updated recently (%s)") % \
3086- format_timedelta(now - t),
3087+ _("The database has been updated recently"),
3088 )
3089 return 3
0be475ca 3090
b952a52b
MT		 3091 # Fetch the timestamp we need from DNS
3092 t = location.discover_latest_version()
0be475ca 3093
b952a52b
MT		 3094- # Parse timestamp into datetime format
3095- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
3096-
3097 # Check the version of the local database
3098- if db and timestamp and db.created_at >= timestamp.timestamp():
3099+ if db and t and db.created_at >= t:
3100 log.info("Already on the latest version")
3101 return
0be475ca 3102
b952a52b 3103@@ -437,7 +433,7 @@ class CLI(object):
0be475ca 3104
b952a52b
MT		 3105 # Try downloading a new database
3106 try:
3107- t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
3108+ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
0be475ca 3109
b952a52b
MT		 3110 # If no file could be downloaded, log a message
3111 except FileNotFoundError as e:
3112@@ -453,13 +449,7 @@ class CLI(object):
0be475ca 3113
b952a52b 3114 return 0
0be475ca 3115
b952a52b
MT		 3116- def handle_verify(self, ns):
3117- try:
3118- db = location.Database(ns.database)
3119- except FileNotFoundError as e:
3120- log.error("%s: %s" % (ns.database, e))
3121- return 127
3122-
3123+ def handle_verify(self, db, ns):
3124 # Verify the database
3125 with open(ns.public_key, "r") as f:
3126 if not db.verify(f):
3127diff --git a/src/python/network.c b/src/python/network.c
3128index 5496d1e..b6e92fb 100644
3129--- a/src/python/network.c
3130+++ b/src/python/network.c
3131@@ -20,10 +20,29 @@
3132
3133 #include <loc/libloc.h>
3134 #include <loc/network.h>
3135+#include <loc/network-list.h>
3136
3137 #include "locationmodule.h"
3138 #include "network.h"
3139
3140+static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
3141+ PyObject* list = PyList_New(0);
3142+ if (!networks)
3143+ return list;
0be475ca 3144+
b952a52b
MT		 3145+ while (!loc_network_list_empty(networks)) {
3146+ struct loc_network* network = loc_network_list_pop(networks);
0be475ca 3147+
b952a52b
MT		 3148+ PyObject* n = new_network(&NetworkType, network);
3149+ PyList_Append(list, n);
0be475ca 3150+
b952a52b
MT		 3151+ loc_network_unref(network);
3152+ Py_DECREF(n);
0be475ca
MT		 3153+ }
3154+
b952a52b
MT		 3155+ return list;
3156+}
0be475ca 3157+
b952a52b
MT		 3158 PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
3159 NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
3160 if (self) {
3161@@ -154,13 +173,28 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
3162 Py_RETURN_NONE;
3163 }
3164
3165+static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
3166+ NetworkObject* other = NULL;
0be475ca 3167+
b952a52b
MT		 3168+ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3169+ return NULL;
0be475ca 3170+
b952a52b 3171+ struct loc_network_list* list = loc_network_exclude(self->network, other->network);
0be475ca 3172+
b952a52b
MT		 3173+ // Convert to Python objects
3174+ PyObject* obj = PyList_FromNetworkList(list);
3175+ loc_network_list_unref(list);
0be475ca 3176+
b952a52b
MT		 3177+ return obj;
3178+}
0be475ca 3179+
b952a52b
MT		 3180 static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
3181 NetworkObject* other = NULL;
0be475ca 3182
b952a52b
MT		 3183 if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3184 return NULL;
0be475ca 3185
b952a52b
MT		 3186- if (loc_network_is_subnet_of(self->network, other->network))
3187+ if (loc_network_is_subnet(other->network, self->network))
3188 Py_RETURN_TRUE;
0be475ca 3189
b952a52b
MT		 3190 Py_RETURN_FALSE;
3191@@ -181,6 +215,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) {
3192 return obj;
0be475ca
MT		 3193 }
3194
b952a52b
MT		 3195+static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) {
3196+ struct in_addr address4;
0be475ca 3197+
b952a52b
MT		 3198+ // Convert IPv4 addresses to struct in_addr
3199+ if (IN6_IS_ADDR_V4MAPPED(address6)) {
3200+ address4.s_addr = address6->s6_addr32[3];
0be475ca 3201+
b952a52b
MT		 3202+ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4));
3203+ }
0be475ca 3204+
b952a52b
MT		 3205+ // Return IPv6 addresses as they are
3206+ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6));
3207+}
0be475ca 3208+
b952a52b
MT		 3209+static PyObject* Network_get__first_address(NetworkObject* self) {
3210+ const struct in6_addr* address = loc_network_get_first_address(self->network);
3211+
3212+ return PyBytes_FromAddress(address);
0be475ca
MT		 3213+}
3214+
b952a52b
MT		 3215 static PyObject* Network_get_last_address(NetworkObject* self) {
3216 char* address = loc_network_format_last_address(self->network);
0be475ca 3217
b952a52b
MT		 3218@@ -190,7 +244,19 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
3219 return obj;
0be475ca 3220 }
0be475ca 3221
b952a52b
MT		 3222+static PyObject* Network_get__last_address(NetworkObject* self) {
3223+ const struct in6_addr* address = loc_network_get_last_address(self->network);
0be475ca 3224+
b952a52b
MT		 3225+ return PyBytes_FromAddress(address);
3226+}
3227+
3228 static struct PyMethodDef Network_methods[] = {
3229+ {
3230+ "exclude",
3231+ (PyCFunction)Network_exclude,
3232+ METH_VARARGS,
3233+ NULL,
3234+ },
3235 {
3236 "has_flag",
3237 (PyCFunction)Network_has_flag,
3238@@ -241,6 +307,13 @@ static struct PyGetSetDef Network_getsetters[] = {
3239 NULL,
3240 NULL,
3241 },
3242+ {
3243+ "_first_address",
3244+ (getter)Network_get__first_address,
3245+ NULL,
3246+ NULL,
3247+ NULL,
3248+ },
3249 {
3250 "last_address",
3251 (getter)Network_get_last_address,
3252@@ -248,6 +321,13 @@ static struct PyGetSetDef Network_getsetters[] = {
3253 NULL,
3254 NULL,
3255 },
3256+ {
3257+ "_last_address",
3258+ (getter)Network_get__last_address,
3259+ NULL,
3260+ NULL,
3261+ NULL,
3262+ },
3263 { NULL },
3264 };
0be475ca 3265
b952a52b
MT		 3266diff --git a/src/test-as.c b/src/test-as.c
3267index 839a04c..2d61675 100644
3268--- a/src/test-as.c
3269+++ b/src/test-as.c
3270@@ -95,7 +95,7 @@ int main(int argc, char** argv) {
3271 // Enumerator
0be475ca 3272
b952a52b
MT		 3273 struct loc_database_enumerator* enumerator;
3274- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
3275+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
3276 if (err) {
3277 fprintf(stderr, "Could not create a database enumerator\n");
3278 exit(EXIT_FAILURE);
3279diff --git a/src/test-database.c b/src/test-database.c
3280index b4a75c4..da4b11c 100644
3281--- a/src/test-database.c
3282+++ b/src/test-database.c
3283@@ -38,6 +38,14 @@ const char* DESCRIPTION =
3284 "Maecenas ut venenatis nunc.";
3285 const char* LICENSE = "CC";
0be475ca 3286
b952a52b
MT		 3287+const char* networks[] = {
3288+ "2001:db8::/32",
3289+ "2001:db8:1000::/48",
3290+ "2001:db8:2000::/48",
3291+ "2001:db8:2020::/48",
3292+ NULL,
3293+};
3294+
3295 static int attempt_to_open(struct loc_ctx* ctx, char* path) {
3296 FILE* f = fopen(path, "r");
3297 if (!f)
3298@@ -139,6 +147,24 @@ int main(int argc, char** argv) {
3299 exit(EXIT_FAILURE);
0be475ca
MT		 3300 }
3301
b952a52b 3302+ struct loc_network* network = NULL;
0be475ca 3303+
b952a52b
MT		 3304+ // Add some networks
3305+ const char** n = networks;
3306+ while (*n) {
3307+ err = loc_writer_add_network(writer, &network, *n);
3308+ if (err) {
3309+ fprintf(stderr, "Could not add network %s\n", *n);
3310+ exit(EXIT_FAILURE);
3311+ }
0be475ca 3312+
b952a52b
MT		 3313+ // Set a country
3314+ loc_network_set_country_code(network, "XX");
0be475ca 3315+
b952a52b
MT		 3316+ // Next one
3317+ n++;
3318+ }
0be475ca 3319+
b952a52b
MT		 3320 FILE* f = tmpfile();
3321 if (!f) {
3322 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
3323@@ -170,6 +196,33 @@ int main(int argc, char** argv) {
3324 exit(EXIT_FAILURE);
0be475ca
MT		 3325 }
3326
b952a52b
MT		 3327+ // Enumerator
3328+ struct loc_database_enumerator* enumerator;
3329+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
3330+ if (err) {
3331+ fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
3332+ exit(EXIT_FAILURE);
3333+ }
0be475ca 3334+
b952a52b
MT		 3335+ // Walk through all networks
3336+ while (1) {
3337+ err = loc_database_enumerator_next_network(enumerator, &network);
3338+ if (err) {
3339+ fprintf(stderr, "Error fetching the next network: %d\n", err);
3340+ exit(EXIT_FAILURE);
3341+ }
3342+
3343+ if (!network)
3344+ break;
3345+
3346+ char* s = loc_network_str(network);
3347+ printf("Got network: %s\n", s);
3348+ free(s);
3349+ }
3350+
3351+ // Free the enumerator
3352+ loc_database_enumerator_unref(enumerator);
3353+
3354 // Close the database
3355 loc_database_unref(db);
3356 loc_unref(ctx);
0be475ca 3357diff --git a/src/test-network-list.c b/src/test-network-list.c
b952a52b
MT		 3358new file mode 100644
3359index 0000000..6f32ff7
3360--- /dev/null
0be475ca 3361+++ b/src/test-network-list.c
b952a52b
MT		 3362@@ -0,0 +1,183 @@
3363+/*
3364+ libloc - A library to determine the location of someone on the Internet
3365+
3366+ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
3367+
3368+ This program is free software; you can redistribute it and/or modify
3369+ it under the terms of the GNU General Public License as published by
3370+ the Free Software Foundation; either version 2 of the License, or
3371+ (at your option) any later version.
3372+
3373+ This program is distributed in the hope that it will be useful,
3374+ but WITHOUT ANY WARRANTY; without even the implied warranty of
3375+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3376+ GNU General Public License for more details.
3377+*/
3378+
3379+#include <errno.h>
3380+#include <stdio.h>
3381+#include <stddef.h>
3382+#include <stdlib.h>
3383+#include <string.h>
3384+#include <syslog.h>
3385+
3386+#include <loc/libloc.h>
3387+#include <loc/network.h>
3388+#include <loc/network-list.h>
3389+
3390+int main(int argc, char** argv) {
3391+ int err;
3392+
3393+ struct loc_ctx* ctx;
3394+ err = loc_new(&ctx);
3395+ if (err < 0)
3396+ exit(EXIT_FAILURE);
3397+
3398+ // Enable debug logging
3399+ loc_set_log_priority(ctx, LOG_DEBUG);
3400+
3401+ // Create a network
3402+ struct loc_network* network1;
3403+ err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32");
3404+ if (err) {
3405+ fprintf(stderr, "Could not create the network1\n");
3406+ exit(EXIT_FAILURE);
3407+ }
3408+
3409+ struct loc_network* subnet1;
3410+ err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48");
3411+ if (err) {
3412+ fprintf(stderr, "Could not create the subnet1\n");
3413+ exit(EXIT_FAILURE);
3414+ }
3415+
3416+ struct loc_network* subnet2;
3417+ err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48");
3418+ if (err) {
3419+ fprintf(stderr, "Could not create the subnet2\n");
3420+ exit(EXIT_FAILURE);
3421+ }
3422+
0be475ca
MT		 3423+ struct loc_network* subnet3;
3424+ err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48");
3425+ if (err) {
3426+ fprintf(stderr, "Could not create the subnet3\n");
3427+ exit(EXIT_FAILURE);
3428+ }
3429+
3430+ struct loc_network* subnet4;
3431+ err = loc_network_new_from_string(ctx, &subnet4, "2001:db8:d::/48");
3432+ if (err) {
3433+ fprintf(stderr, "Could not create the subnet4\n");
3434+ exit(EXIT_FAILURE);
3435+ }
3436+
b952a52b
MT		 3437+ struct loc_network* subnet5;
3438+ err = loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48");
3439+ if (err) {
3440+ fprintf(stderr, "Could not create the subnet5\n");
3441+ exit(EXIT_FAILURE);
3442+ }
3443+
3444+ struct loc_network* subnet6;
3445+ err = loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48");
3446+ if (err) {
3447+ fprintf(stderr, "Could not create the subnet6\n");
3448+ exit(EXIT_FAILURE);
3449+ }
3450+
3451+ // Make a list with both subnets
3452+ struct loc_network_list* subnets;
3453+ err = loc_network_list_new(ctx, &subnets);
3454+ if (err) {
3455+ fprintf(stderr, "Could not create subnets list\n");
3456+ exit(EXIT_FAILURE);
3457+ }
3458+
3459+ size_t size = loc_network_list_size(subnets);
3460+ if (size > 0) {
3461+ fprintf(stderr, "The list is not empty: %zu\n", size);
3462+ exit(EXIT_FAILURE);
3463+ }
3464+
3465+ err = loc_network_list_push(subnets, subnet1);
3466+ if (err) {
3467+ fprintf(stderr, "Could not add subnet1 to subnets list\n");
3468+ exit(EXIT_FAILURE);
3469+ }
3470+
3471+ if (loc_network_list_empty(subnets)) {
3472+ fprintf(stderr, "The subnets list reports that it is empty\n");
3473+ exit(EXIT_FAILURE);
3474+ }
3475+
3476+ err = loc_network_list_push(subnets, subnet2);
3477+ if (err) {
3478+ fprintf(stderr, "Could not add subnet2 to subnets list\n");
3479+ exit(EXIT_FAILURE);
3480+ }
3481+
3482+ // Add the fourth one next
3483+ err = loc_network_list_push(subnets, subnet4);
3484+ if (err) {
3485+ fprintf(stderr, "Could not add subnet4 to subnets list\n");
3486+ exit(EXIT_FAILURE);
3487+ }
3488+
3489+ // Add the third one
3490+ err = loc_network_list_push(subnets, subnet3);
3491+ if (err) {
3492+ fprintf(stderr, "Could not add subnet3 to subnets list\n");
3493+ exit(EXIT_FAILURE);
3494+ }
3495+
3496+ // Add more subnets
3497+ err = loc_network_list_push(subnets, subnet5);
3498+ if (err) {
3499+ fprintf(stderr, "Could not add subnet5 to subnets list\n");
3500+ exit(EXIT_FAILURE);
3501+ }
3502+
3503+ err = loc_network_list_push(subnets, subnet6);
3504+ if (err) {
3505+ fprintf(stderr, "Could not add subnet6 to subnets list\n");
3506+ exit(EXIT_FAILURE);
3507+ }
3508+
3509+ loc_network_list_dump(subnets);
3510+
3511+ size = loc_network_list_size(subnets);
3512+ if (size != 6) {
3513+ fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
3514+ exit(EXIT_FAILURE);
3515+ }
3516+
3517+ // Exclude subnet1 from network1
3518+ struct loc_network_list* excluded = loc_network_exclude(network1, subnet1);
3519+ if (!excluded) {
3520+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n");
0be475ca
MT		 3521+ exit(EXIT_FAILURE);
3522+ }
3523+
b952a52b
MT		 3524+ loc_network_list_dump(excluded);
3525+
3526+ // Exclude all subnets from network1
3527+ excluded = loc_network_exclude_list(network1, subnets);
3528+ if (!excluded) {
3529+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n");
0be475ca
MT		 3530+ exit(EXIT_FAILURE);
3531+ }
3532+
b952a52b 3533+ loc_network_list_dump(excluded);
0be475ca 3534+
b952a52b
MT		 3535+ if (excluded)
3536+ loc_network_list_unref(excluded);
3537+
3538+ loc_network_list_unref(subnets);
3539+ loc_network_unref(network1);
3540+ loc_network_unref(subnet1);
3541+ loc_network_unref(subnet2);
3542+ loc_unref(ctx);
3543+
3544+ return EXIT_SUCCESS;
3545+}
0be475ca 3546diff --git a/src/test-network.c b/src/test-network.c
b952a52b 3547index d38f13d..dde13f1 100644
0be475ca
MT		 3548--- a/src/test-network.c
3549+++ b/src/test-network.c
3550@@ -14,6 +14,7 @@
3551 GNU General Public License for more details.
3552 */
3553
3554+#include <arpa/inet.h>
3555 #include <errno.h>
3556 #include <stdio.h>
3557 #include <stddef.h>
b952a52b
MT		 3558@@ -37,12 +38,21 @@ int main(int argc, char** argv) {
3559 // Enable debug logging
3560 loc_set_log_priority(ctx, LOG_DEBUG);
0be475ca 3561
b952a52b
MT		 3562+#if 0
3563 struct loc_network_tree* tree;
3564 err = loc_network_tree_new(ctx, &tree);
3565 if (err) {
3566 fprintf(stderr, "Could not create the network tree\n");
3567 exit(EXIT_FAILURE);
3568 }
3569+#endif
3570+
0be475ca
MT		 3571+ struct in6_addr address;
3572+ err = inet_pton(AF_INET6, "2001:db8::1", &address);
3573+ if (err != 1) {
3574+ fprintf(stderr, "Could not parse IP address\n");
3575+ exit(EXIT_FAILURE);
3576+ }
b952a52b 3577
0be475ca
MT		 3578 // Create a network
3579 struct loc_network* network1;
b952a52b 3580@@ -58,12 +68,14 @@ int main(int argc, char** argv) {
0be475ca
MT		 3581 exit(EXIT_FAILURE);
3582 }
3583
b952a52b
MT		 3584+#if 0
3585 // Adding network to the tree
3586 err = loc_network_tree_add_network(tree, network1);
0be475ca 3587 if (err) {
b952a52b
MT		 3588 fprintf(stderr, "Could not add network to the tree\n");
3589 exit(EXIT_FAILURE);
0be475ca 3590 }
b952a52b 3591+#endif
0be475ca 3592
b952a52b
MT		 3593 // Check if the first and last addresses are correct
3594 char* string = loc_network_format_first_address(network1);
3595@@ -88,6 +100,12 @@ int main(int argc, char** argv) {
3596 exit(EXIT_FAILURE);
0be475ca
MT		 3597 }
3598
b952a52b
MT		 3599+ err = loc_network_match_address(network1, &address);
3600+ if (!err) {
3601+ fprintf(stderr, "Network1 does not match address\n");
3602+ exit(EXIT_FAILURE);
3603+ }
0be475ca 3604+
b952a52b
MT		 3605 struct loc_network* network2;
3606 err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48");
3607 if (err) {
3608@@ -101,6 +119,7 @@ int main(int argc, char** argv) {
3609 exit(EXIT_FAILURE);
3610 }
0be475ca 3611
b952a52b
MT		 3612+#if 0
3613 // Adding network to the tree
3614 err = loc_network_tree_add_network(tree, network2);
3615 if (err) {
3616@@ -117,20 +136,84 @@ int main(int argc, char** argv) {
0be475ca 3617
b952a52b
MT		 3618 size_t nodes = loc_network_tree_count_nodes(tree);
3619 printf("The tree has %zu nodes\n", nodes);
3620+#endif
3621+
3622+ // Check equals function
3623+ err = loc_network_cmp(network1, network1);
3624+ if (err) {
3625+ fprintf(stderr, "Network is not equal with itself\n");
3626+ exit(EXIT_FAILURE);
3627+ }
3628+
3629+ err = loc_network_cmp(network1, network2);
3630+ if (!err) {
3631+ fprintf(stderr, "Networks equal unexpectedly\n");
3632+ exit(EXIT_FAILURE);
3633+ }
3634
3635 // Check subnet function
3636- err = loc_network_is_subnet_of(network1, network2);
3637- if (err != 0) {
3638+ err = loc_network_is_subnet(network1, network2);
3639+ if (!err) {
3640 fprintf(stderr, "Subnet check 1 failed: %d\n", err);
3641 exit(EXIT_FAILURE);
0be475ca 3642 }
0be475ca 3643
b952a52b
MT		 3644- err = loc_network_is_subnet_of(network2, network1);
3645- if (err != 1) {
3646+ err = loc_network_is_subnet(network2, network1);
3647+ if (err) {
3648 fprintf(stderr, "Subnet check 2 failed: %d\n", err);
0be475ca
MT		 3649 exit(EXIT_FAILURE);
3650 }
3651
b952a52b
MT		 3652+ // Make subnets
3653+ struct loc_network* subnet1 = NULL;
3654+ struct loc_network* subnet2 = NULL;
3655+
3656+ err = loc_network_subnets(network1, &subnet1, &subnet2);
3657+ if (err || !subnet1 || !subnet2) {
3658+ fprintf(stderr, "Could not find subnets of network: %d\n", err);
3659+ exit(EXIT_FAILURE);
3660+ }
3661+
3662+ char* s = loc_network_str(subnet1);
3663+ printf("Received subnet1 = %s\n", s);
3664+ free(s);
3665+
3666+ s = loc_network_str(subnet2);
3667+ printf("Received subnet2 = %s\n", s);
3668+ free(s);
3669+
3670+ if (!loc_network_is_subnet(network1, subnet1)) {
3671+ fprintf(stderr, "Subnet1 is not a subnet\n");
3672+ exit(EXIT_FAILURE);
3673+ }
3674+
3675+ if (!loc_network_is_subnet(network1, subnet2)) {
3676+ fprintf(stderr, "Subnet2 is not a subnet\n");
3677+ exit(EXIT_FAILURE);
3678+ }
3679+
0be475ca
MT		 3680+ if (!loc_network_overlaps(network1, subnet1)) {
3681+ fprintf(stderr, "Network1 does not seem to contain subnet1\n");
3682+ exit(EXIT_FAILURE);
3683+ }
3684+
3685+ if (!loc_network_overlaps(network1, subnet2)) {
3686+ fprintf(stderr, "Network1 does not seem to contain subnet2\n");
3687+ exit(EXIT_FAILURE);
3688+ }
3689+
b952a52b
MT		 3690+ loc_network_unref(subnet1);
3691+ loc_network_unref(subnet2);
0be475ca 3692+
b952a52b
MT		 3693+ struct loc_network_list* excluded = loc_network_exclude(network1, network2);
3694+ if (!excluded) {
3695+ fprintf(stderr, "Could not create excluded list\n");
3696+ exit(EXIT_FAILURE);
3697+ }
0be475ca 3698+
b952a52b
MT		 3699+ loc_network_list_dump(excluded);
3700+ loc_network_list_unref(excluded);
0be475ca 3701+
b952a52b
MT		 3702 // Create a database
3703 struct loc_writer* writer;
3704 err = loc_writer_new(ctx, &writer, NULL, NULL);
3705@@ -160,6 +243,28 @@ int main(int argc, char** argv) {
3706 // Set ASN
3707 loc_network_set_asn(network4, 1024);
0be475ca 3708
b952a52b
MT		 3709+ // Try adding an invalid network
3710+ struct loc_network* network;
3711+ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
3712+ if (err != -EINVAL) {
3713+ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
3714+ exit(EXIT_FAILURE);
3715+ }
3716+
3717+ // Try adding a single address
3718+ err = loc_writer_add_network(writer, &network, "2001:db8::");
3719+ if (err) {
3720+ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
3721+ exit(EXIT_FAILURE);
3722+ }
3723+
3724+ // Try adding localhost
3725+ err = loc_writer_add_network(writer, &network, "::1/128");
3726+ if (err != -EINVAL) {
3727+ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
3728+ exit(EXIT_FAILURE);
3729+ }
3730+
3731 FILE* f = tmpfile();
3732 if (!f) {
3733 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
3734@@ -177,7 +282,10 @@ int main(int argc, char** argv) {
3735 loc_network_unref(network2);
3736 loc_network_unref(network3);
3737 loc_network_unref(network4);
3738+
3739+#if 0
3740 loc_network_tree_unref(tree);
3741+#endif
0be475ca 3742
b952a52b
MT		 3743 // And open it again from disk
3744 struct loc_database* db;
3745diff --git a/src/writer.c b/src/writer.c
3746index 5939cff..2f09b56 100644
3747--- a/src/writer.c
3748+++ b/src/writer.c
3749@@ -147,8 +147,19 @@ static void loc_writer_free(struct loc_writer* writer) {
3750 EVP_PKEY_free(writer->private_key2);
0be475ca 3751
b952a52b
MT		 3752 // Unref all AS
3753- for (unsigned int i = 0; i < writer->as_count; i++) {
3754- loc_as_unref(writer->as[i]);
3755+ if (writer->as) {
3756+ for (unsigned int i = 0; i < writer->as_count; i++) {
3757+ loc_as_unref(writer->as[i]);
3758+ }
3759+ free(writer->as);
3760+ }
0be475ca 3761+
b952a52b
MT		 3762+ // Unref all countries
3763+ if (writer->countries) {
3764+ for (unsigned int i = 0; i < writer->countries_count; i++) {
3765+ loc_country_unref(writer->countries[i]);
3766+ }
3767+ free(writer->countries);
3768 }
0be475ca 3769
b952a52b 3770 // Release network tree
Peter's tree of IPFire 2.x