]> git.ipfire.org Git - people/pmueller/ipfire-2.x.git/blob - src/patches/libloc-0.9.4-upstream.patch
projects / people / pmueller / ipfire-2.x.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
libloc: Import changes from upstream
[people/pmueller/ipfire-2.x.git] / src / patches / libloc-0.9.4-upstream.patch
1 diff --git a/Makefile.am b/Makefile.am
2 index a0431a6..dc594f8 100644
3 --- a/Makefile.am
4 +++ b/Makefile.am
5 @@ -91,11 +91,14 @@ EXTRA_DIST += \
6 pkginclude_HEADERS = \
7 src/loc/libloc.h \
8 src/loc/as.h \
9 + src/loc/as-list.h \
10 src/loc/compat.h \
11 src/loc/country.h \
12 + src/loc/country-list.h \
13 src/loc/database.h \
14 src/loc/format.h \
15 src/loc/network.h \
16 + src/loc/network-list.h \
17 src/loc/private.h \
18 src/loc/stringpool.h \
19 src/loc/resolv.h \
20 @@ -107,9 +110,12 @@ lib_LTLIBRARIES = \
21 src_libloc_la_SOURCES = \
22 src/libloc.c \
23 src/as.c \
24 + src/as-list.c \
25 src/country.c \
26 + src/country-list.c \
27 src/database.c \
28 src/network.c \
29 + src/network-list.c \
30 src/resolv.c \
31 src/stringpool.c \
32 src/writer.c
33 @@ -312,6 +318,7 @@ check_PROGRAMS = \
34 src/test-database \
35 src/test-as \
36 src/test-network \
37 + src/test-network-list \
38 src/test-country \
39 src/test-signature
40
41 @@ -351,6 +358,15 @@ src_test_network_CFLAGS = \
42 src_test_network_LDADD = \
43 src/libloc.la
44
45 +src_test_network_list_SOURCES = \
46 + src/test-network-list.c
47 +
48 +src_test_network_list_CFLAGS = \
49 + $(TESTS_CFLAGS)
50 +
51 +src_test_network_list_LDADD = \
52 + src/libloc.la
53 +
54 src_test_stringpool_SOURCES = \
55 src/test-stringpool.c
56
57 @@ -390,7 +406,7 @@ MANPAGES_XML = $(patsubst %.txt,%.xml,$(MANPAGES_TXT))
58 .PHONY: man
59 man: $(MANPAGES) $(MANPAGES_HTML)
60
61 -if ENABLE_MANPAGES
62 +if ENABLE_MAN_PAGES
63 man_MANS = \
64 $(MANPAGES)
65 endif
66 diff --git a/configure.ac b/configure.ac
67 index 2364dfd..9eb9012 100644
68 --- a/configure.ac
69 +++ b/configure.ac
70 @@ -1,6 +1,6 @@
71 AC_PREREQ(2.60)
72 AC_INIT([libloc],
73 - [0.9.4],
74 + [0.9.5],
75 [location@lists.ipfire.org],
76 [libloc],
77 [https://location.ipfire.org/])
78 @@ -43,16 +43,16 @@ AC_PROG_MKDIR_P
79
80 # - man ------------------------------------------------------------------------
81
82 -have_manpages=no
83 -AC_ARG_ENABLE(manpages, AS_HELP_STRING([--disable-man-pages],
84 +have_man_pages=no
85 +AC_ARG_ENABLE(man_pages, AS_HELP_STRING([--disable-man-pages],
86 [do not install man pages]))
87 -AS_IF([test "x$enable_manpages" != xno], [have_manpages=yes])
88 -AM_CONDITIONAL(ENABLE_MANPAGES, [test "x$have_manpages" = "xyes"])
89 +AS_IF([test "x$enable_man_pages" != xno], [have_man_pages=yes])
90 +AM_CONDITIONAL(ENABLE_MAN_PAGES, [test "x$have_man_pages" = "xyes"])
91
92 AC_PATH_PROG([XSLTPROC], [xsltproc])
93
94 AC_CHECK_PROGS(ASCIIDOC, [asciidoc])
95 -if test "${have_manpages}" = "yes" && test -z "${ASCIIDOC}"; then
96 +if test "${have_man_pages}" = "yes" && test -z "${ASCIIDOC}"; then
97 AC_MSG_ERROR([Required program 'asciidoc' not found])
98 fi
99 # - debug ----------------------------------------------------------------------
100 diff --git a/src/.gitignore b/src/.gitignore
101 index caf80b5..3ccbdb8 100644
102 --- a/src/.gitignore
103 +++ b/src/.gitignore
104 @@ -10,5 +10,6 @@ test-libloc
105 test-database
106 test-country
107 test-network
108 +test-network-list
109 test-signature
110 test-stringpool
111 diff --git a/src/as-list.c b/src/as-list.c
112 new file mode 100644
113 index 0000000..5acbb8a
114 --- /dev/null
115 +++ b/src/as-list.c
116 @@ -0,0 +1,161 @@
117 +/*
118 + libloc - A library to determine the location of someone on the Internet
119 +
120 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
121 +
122 + This library is free software; you can redistribute it and/or
123 + modify it under the terms of the GNU Lesser General Public
124 + License as published by the Free Software Foundation; either
125 + version 2.1 of the License, or (at your option) any later version.
126 +
127 + This library is distributed in the hope that it will be useful,
128 + but WITHOUT ANY WARRANTY; without even the implied warranty of
129 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
130 + Lesser General Public License for more details.
131 +*/
132 +
133 +#include <errno.h>
134 +#include <stdlib.h>
135 +
136 +#include <loc/as.h>
137 +#include <loc/as-list.h>
138 +#include <loc/private.h>
139 +
140 +struct loc_as_list {
141 + struct loc_ctx* ctx;
142 + int refcount;
143 +
144 + struct loc_as** elements;
145 + size_t elements_size;
146 +
147 + size_t size;
148 +};
149 +
150 +static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
151 + DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
152 + list, size, list->elements_size + size);
153 +
154 + struct loc_as** elements = reallocarray(list->elements,
155 + list->elements_size + size, sizeof(*list->elements));
156 + if (!elements)
157 + return -errno;
158 +
159 + list->elements = elements;
160 + list->elements_size += size;
161 +
162 + return 0;
163 +}
164 +
165 +LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
166 + struct loc_as_list** list) {
167 + struct loc_as_list* l = calloc(1, sizeof(*l));
168 + if (!l)
169 + return -ENOMEM;
170 +
171 + l->ctx = loc_ref(ctx);
172 + l->refcount = 1;
173 +
174 + DEBUG(l->ctx, "AS list allocated at %p\n", l);
175 + *list = l;
176 +
177 + return 0;
178 +}
179 +
180 +LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
181 + list->refcount++;
182 +
183 + return list;
184 +}
185 +
186 +static void loc_as_list_free(struct loc_as_list* list) {
187 + DEBUG(list->ctx, "Releasing AS list at %p\n", list);
188 +
189 + loc_as_list_clear(list);
190 +
191 + loc_unref(list->ctx);
192 + free(list);
193 +}
194 +
195 +LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
196 + if (!list)
197 + return NULL;
198 +
199 + if (--list->refcount > 0)
200 + return list;
201 +
202 + loc_as_list_free(list);
203 + return NULL;
204 +}
205 +
206 +LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
207 + return list->size;
208 +}
209 +
210 +LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
211 + return list->size == 0;
212 +}
213 +
214 +LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
215 + if (!list->elements)
216 + return;
217 +
218 + for (unsigned int i = 0; i < list->size; i++)
219 + loc_as_unref(list->elements[i]);
220 +
221 + free(list->elements);
222 + list->elements = NULL;
223 + list->elements_size = 0;
224 +
225 + list->size = 0;
226 +}
227 +
228 +LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
229 + // Check index
230 + if (index >= list->size)
231 + return NULL;
232 +
233 + return loc_as_ref(list->elements[index]);
234 +}
235 +
236 +LOC_EXPORT int loc_as_list_append(
237 + struct loc_as_list* list, struct loc_as* as) {
238 + if (loc_as_list_contains(list, as))
239 + return 0;
240 +
241 + // Check if we have space left
242 + if (list->size >= list->elements_size) {
243 + int r = loc_as_list_grow(list, 64);
244 + if (r)
245 + return r;
246 + }
247 +
248 + DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
249 +
250 + list->elements[list->size++] = loc_as_ref(as);
251 +
252 + return 0;
253 +}
254 +
255 +LOC_EXPORT int loc_as_list_contains(
256 + struct loc_as_list* list, struct loc_as* as) {
257 + for (unsigned int i = 0; i < list->size; i++) {
258 + if (loc_as_cmp(as, list->elements[i]) == 0)
259 + return 1;
260 + }
261 +
262 + return 0;
263 +}
264 +
265 +LOC_EXPORT int loc_as_list_contains_number(
266 + struct loc_as_list* list, uint32_t number) {
267 + struct loc_as* as;
268 +
269 + int r = loc_as_new(list->ctx, &as, number);
270 + if (r)
271 + return -1;
272 +
273 + r = loc_as_list_contains(list, as);
274 + loc_as_unref(as);
275 +
276 + return r;
277 +}
278 diff --git a/src/as.c b/src/as.c
279 index e1fbb01..757bf3d 100644
280 --- a/src/as.c
281 +++ b/src/as.c
282 @@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
283 }
284
285 LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) {
286 - as->name = strdup(name);
287 + if (as->name)
288 + free(as->name);
289 +
290 + if (name)
291 + as->name = strdup(name);
292 + else
293 + as->name = NULL;
294
295 return 0;
296 }
297 @@ -139,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
298 if (!string)
299 return 1;
300
301 + // Cannot match anything when name is not set
302 + if (!as->name)
303 + return 1;
304 +
305 // Search if string is in name
306 if (strcasestr(as->name, string) != NULL)
307 return 1;
308 diff --git a/src/country-list.c b/src/country-list.c
309 new file mode 100644
310 index 0000000..cc36740
311 --- /dev/null
312 +++ b/src/country-list.c
313 @@ -0,0 +1,161 @@
314 +/*
315 + libloc - A library to determine the location of someone on the Internet
316 +
317 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
318 +
319 + This library is free software; you can redistribute it and/or
320 + modify it under the terms of the GNU Lesser General Public
321 + License as published by the Free Software Foundation; either
322 + version 2.1 of the License, or (at your option) any later version.
323 +
324 + This library is distributed in the hope that it will be useful,
325 + but WITHOUT ANY WARRANTY; without even the implied warranty of
326 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
327 + Lesser General Public License for more details.
328 +*/
329 +
330 +#include <errno.h>
331 +#include <stdlib.h>
332 +
333 +#include <loc/country.h>
334 +#include <loc/country-list.h>
335 +#include <loc/private.h>
336 +
337 +struct loc_country_list {
338 + struct loc_ctx* ctx;
339 + int refcount;
340 +
341 + struct loc_country** elements;
342 + size_t elements_size;
343 +
344 + size_t size;
345 +};
346 +
347 +static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
348 + DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
349 + list, size, list->elements_size + size);
350 +
351 + struct loc_country** elements = reallocarray(list->elements,
352 + list->elements_size + size, sizeof(*list->elements));
353 + if (!elements)
354 + return -errno;
355 +
356 + list->elements = elements;
357 + list->elements_size += size;
358 +
359 + return 0;
360 +}
361 +
362 +LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
363 + struct loc_country_list** list) {
364 + struct loc_country_list* l = calloc(1, sizeof(*l));
365 + if (!l)
366 + return -ENOMEM;
367 +
368 + l->ctx = loc_ref(ctx);
369 + l->refcount = 1;
370 +
371 + DEBUG(l->ctx, "Country list allocated at %p\n", l);
372 + *list = l;
373 +
374 + return 0;
375 +}
376 +
377 +LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
378 + list->refcount++;
379 +
380 + return list;
381 +}
382 +
383 +static void loc_country_list_free(struct loc_country_list* list) {
384 + DEBUG(list->ctx, "Releasing country list at %p\n", list);
385 +
386 + loc_country_list_clear(list);
387 +
388 + loc_unref(list->ctx);
389 + free(list);
390 +}
391 +
392 +LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
393 + if (!list)
394 + return NULL;
395 +
396 + if (--list->refcount > 0)
397 + return list;
398 +
399 + loc_country_list_free(list);
400 + return NULL;
401 +}
402 +
403 +LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
404 + return list->size;
405 +}
406 +
407 +LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
408 + return list->size == 0;
409 +}
410 +
411 +LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
412 + if (!list->elements)
413 + return;
414 +
415 + for (unsigned int i = 0; i < list->size; i++)
416 + loc_country_unref(list->elements[i]);
417 +
418 + free(list->elements);
419 + list->elements = NULL;
420 + list->elements_size = 0;
421 +
422 + list->size = 0;
423 +}
424 +
425 +LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
426 + // Check index
427 + if (index >= list->size)
428 + return NULL;
429 +
430 + return loc_country_ref(list->elements[index]);
431 +}
432 +
433 +LOC_EXPORT int loc_country_list_append(
434 + struct loc_country_list* list, struct loc_country* country) {
435 + if (loc_country_list_contains(list, country))
436 + return 0;
437 +
438 + // Check if we have space left
439 + if (list->size >= list->elements_size) {
440 + int r = loc_country_list_grow(list, 64);
441 + if (r)
442 + return r;
443 + }
444 +
445 + DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
446 +
447 + list->elements[list->size++] = loc_country_ref(country);
448 +
449 + return 0;
450 +}
451 +
452 +LOC_EXPORT int loc_country_list_contains(
453 + struct loc_country_list* list, struct loc_country* country) {
454 + for (unsigned int i = 0; i < list->size; i++) {
455 + if (loc_country_cmp(country, list->elements[i]) == 0)
456 + return 1;
457 + }
458 +
459 + return 0;
460 +}
461 +
462 +LOC_EXPORT int loc_country_list_contains_code(
463 + struct loc_country_list* list, const char* code) {
464 + struct loc_country* country;
465 +
466 + int r = loc_country_new(list->ctx, &country, code);
467 + if (r)
468 + return -1;
469 +
470 + r = loc_country_list_contains(list, country);
471 + loc_country_unref(country);
472 +
473 + return r;
474 +}
475 diff --git a/src/country.c b/src/country.c
476 index 2ba93e6..7aac0db 100644
477 --- a/src/country.c
478 +++ b/src/country.c
479 @@ -34,6 +34,9 @@ struct loc_country {
480 };
481
482 LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
483 + if (!loc_country_code_is_valid(country_code))
484 + return -EINVAL;
485 +
486 struct loc_country* c = calloc(1, sizeof(*c));
487 if (!c)
488 return -ENOMEM;
489 diff --git a/src/database.c b/src/database.c
490 index fa1dad0..7100298 100644
491 --- a/src/database.c
492 +++ b/src/database.c
493 @@ -38,8 +38,10 @@
494
495 #include <loc/libloc.h>
496 #include <loc/as.h>
497 +#include <loc/as-list.h>
498 #include <loc/compat.h>
499 #include <loc/country.h>
500 +#include <loc/country-list.h>
501 #include <loc/database.h>
502 #include <loc/format.h>
503 #include <loc/network.h>
504 @@ -99,11 +101,14 @@ struct loc_database_enumerator {
505
506 // Search string
507 char* string;
508 - char country_code[3];
509 - uint32_t asn;
510 + struct loc_country_list* countries;
511 + struct loc_as_list* asns;
512 enum loc_network_flags flags;
513 int family;
514
515 + // Flatten output?
516 + int flatten;
517 +
518 // Index of the AS we are looking at
519 unsigned int as_index;
520
521 @@ -115,6 +120,9 @@ struct loc_database_enumerator {
522 struct loc_node_stack network_stack[MAX_STACK_DEPTH];
523 int network_stack_depth;
524 unsigned int* networks_visited;
525 +
526 + // For subnet search
527 + struct loc_network_list* stack;
528 };
529
530 static int loc_database_read_magic(struct loc_database* db) {
531 @@ -242,11 +250,11 @@ static int loc_database_read_signature(struct loc_database* db,
532 char** dst, char* src, size_t length) {
533 // Check for a plausible signature length
534 if (length > LOC_SIGNATURE_MAX_LENGTH) {
535 - ERROR(db->ctx, "Signature too long: %ld\n", length);
536 + ERROR(db->ctx, "Signature too long: %zu\n", length);
537 return -EINVAL;
538 }
539
540 - DEBUG(db->ctx, "Reading signature of %ld bytes\n", length);
541 + DEBUG(db->ctx, "Reading signature of %zu bytes\n", length);
542
543 // Allocate space
544 *dst = malloc(length);
545 @@ -611,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
546 }
547
548 clock_t end = clock();
549 - DEBUG(db->ctx, "Signature checked in %.4fms\n",
550 + INFO(db->ctx, "Signature checked in %.4fms\n",
551 (double)(end - start) / CLOCKS_PER_SEC * 1000);
552
553 CLEANUP:
554 @@ -671,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
555 off_t lo = 0;
556 off_t hi = db->as_count - 1;
557
558 +#ifdef ENABLE_DEBUG
559 // Save start time
560 clock_t start = clock();
561 +#endif
562
563 while (lo <= hi) {
564 off_t i = (lo + hi) / 2;
565 @@ -685,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
566 // Check if this is a match
567 uint32_t as_number = loc_as_get_number(*as);
568 if (as_number == number) {
569 +#ifdef ENABLE_DEBUG
570 clock_t end = clock();
571
572 // Log how fast this has been
573 DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
574 (double)(end - start) / CLOCKS_PER_SEC * 1000);
575 +#endif
576
577 return 0;
578 }
579 @@ -733,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
580 return -1;
581 }
582
583 +#ifdef ENABLE_DEBUG
584 if (r == 0) {
585 char* string = loc_network_str(*network);
586 DEBUG(db->ctx, "Got network %s\n", string);
587 free(string);
588 }
589 +#endif
590
591 return r;
592 }
593 @@ -762,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru
594 }
595
596 // Check if the given IP address is inside the network
597 - r = loc_network_match_address(*network, address);
598 - if (r) {
599 + if (!loc_network_match_address(*network, address)) {
600 DEBUG(db->ctx, "Searched address is not part of the network\n");
601
602 loc_network_unref(*network);
603 @@ -832,17 +845,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
604
605 *network = NULL;
606
607 +#ifdef ENABLE_DEBUG
608 // Save start time
609 clock_t start = clock();
610 +#endif
611
612 int r = __loc_database_lookup(db, address, network, &network_address,
613 db->network_nodes_v1, 0);
614
615 +#ifdef ENABLE_DEBUG
616 clock_t end = clock();
617
618 // Log how fast this has been
619 DEBUG(db->ctx, "Executed network search in %.4fms\n",
620 (double)(end - start) / CLOCKS_PER_SEC * 1000);
621 +#endif
622
623 return r;
624 }
625 @@ -889,8 +906,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
626 off_t lo = 0;
627 off_t hi = db->countries_count - 1;
628
629 +#ifdef ENABLE_DEBUG
630 // Save start time
631 clock_t start = clock();
632 +#endif
633
634 while (lo <= hi) {
635 off_t i = (lo + hi) / 2;
636 @@ -905,11 +924,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
637 int result = strcmp(code, cc);
638
639 if (result == 0) {
640 +#ifdef ENABLE_DEBUG
641 clock_t end = clock();
642
643 // Log how fast this has been
644 DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
645 (double)(end - start) / CLOCKS_PER_SEC * 1000);
646 +#endif
647
648 return 0;
649 }
650 @@ -932,8 +953,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
651
652 // Enumerator
653
654 +static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
655 + DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
656 +
657 + // Release all references
658 + loc_database_unref(enumerator->db);
659 + loc_unref(enumerator->ctx);
660 +
661 + if (enumerator->string)
662 + free(enumerator->string);
663 +
664 + if (enumerator->countries)
665 + loc_country_list_unref(enumerator->countries);
666 +
667 + if (enumerator->asns)
668 + loc_as_list_unref(enumerator->asns);
669 +
670 + // Free network search
671 + free(enumerator->networks_visited);
672 +
673 + // Free subnet stack
674 + if (enumerator->stack)
675 + loc_network_list_unref(enumerator->stack);
676 +
677 + free(enumerator);
678 +}
679 +
680 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
681 - struct loc_database* db, enum loc_database_enumerator_mode mode) {
682 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
683 struct loc_database_enumerator* e = calloc(1, sizeof(*e));
684 if (!e)
685 return -ENOMEM;
686 @@ -944,11 +991,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
687 e->mode = mode;
688 e->refcount = 1;
689
690 + // Flatten output?
691 + e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
692 +
693 // Initialise graph search
694 - //e->network_stack[++e->network_stack_depth] = 0;
695 e->network_stack_depth = 1;
696 e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
697
698 + // Allocate stack
699 + int r = loc_network_list_new(e->ctx, &e->stack);
700 + if (r) {
701 + loc_database_enumerator_free(e);
702 + return r;
703 + }
704 +
705 DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
706
707 *enumerator = e;
708 @@ -961,22 +1017,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
709 return enumerator;
710 }
711
712 -static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
713 - DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
714 -
715 - // Release all references
716 - loc_database_unref(enumerator->db);
717 - loc_unref(enumerator->ctx);
718 -
719 - if (enumerator->string)
720 - free(enumerator->string);
721 -
722 - // Free network search
723 - free(enumerator->networks_visited);
724 -
725 - free(enumerator);
726 -}
727 -
728 LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
729 if (!enumerator)
730 return NULL;
731 @@ -998,40 +1038,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
732 return 0;
733 }
734
735 -LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
736 - // Set empty country code
737 - if (!country_code || !*country_code) {
738 - *enumerator->country_code = '\0';
739 - return 0;
740 - }
741 +LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
742 + struct loc_database_enumerator* enumerator) {
743 + if (!enumerator->countries)
744 + return NULL;
745
746 - // Treat A1, A2, A3 as special country codes,
747 - // but perform search for flags instead
748 - if (strcmp(country_code, "A1") == 0) {
749 - return loc_database_enumerator_set_flag(enumerator,
750 - LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
751 - } else if (strcmp(country_code, "A2") == 0) {
752 - return loc_database_enumerator_set_flag(enumerator,
753 - LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
754 - } else if (strcmp(country_code, "A3") == 0) {
755 - return loc_database_enumerator_set_flag(enumerator,
756 - LOC_NETWORK_FLAG_ANYCAST);
757 - }
758 + return loc_country_list_ref(enumerator->countries);
759 +}
760
761 - // Country codes must be two characters
762 - if (!loc_country_code_is_valid(country_code))
763 - return -EINVAL;
764 +LOC_EXPORT int loc_database_enumerator_set_countries(
765 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
766 + if (enumerator->countries)
767 + loc_country_list_unref(enumerator->countries);
768
769 - for (unsigned int i = 0; i < 3; i++) {
770 - enumerator->country_code[i] = country_code[i];
771 - }
772 + enumerator->countries = loc_country_list_ref(countries);
773
774 return 0;
775 }
776
777 -LOC_EXPORT int loc_database_enumerator_set_asn(
778 - struct loc_database_enumerator* enumerator, unsigned int asn) {
779 - enumerator->asn = asn;
780 +LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
781 + struct loc_database_enumerator* enumerator) {
782 + if (!enumerator->asns)
783 + return NULL;
784 +
785 + return loc_as_list_ref(enumerator->asns);
786 +}
787 +
788 +LOC_EXPORT int loc_database_enumerator_set_asns(
789 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
790 + if (enumerator->asns)
791 + loc_as_list_unref(enumerator->asns);
792 +
793 + enumerator->asns = loc_as_list_ref(asns);
794
795 return 0;
796 }
797 @@ -1110,16 +1148,64 @@ static int loc_database_enumerator_stack_push_node(
798 return 0;
799 }
800
801 -LOC_EXPORT int loc_database_enumerator_next_network(
802 - struct loc_database_enumerator* enumerator, struct loc_network** network) {
803 - // Reset network
804 - *network = NULL;
805 +static int loc_database_enumerator_filter_network(
806 + struct loc_database_enumerator* enumerator, struct loc_network* network) {
807 + // Skip if the family does not match
808 + if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
809 + DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
810 + return 1;
811 + }
812
813 - // Do not do anything if not in network mode
814 - if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
815 - return 0;
816 + // Skip if the country code does not match
817 + if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
818 + const char* country_code = loc_network_get_country_code(network);
819
820 - int r;
821 + if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
822 + DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
823 + return 1;
824 + }
825 + }
826 +
827 + // Skip if the ASN does not match
828 + if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
829 + uint32_t asn = loc_network_get_asn(network);
830 +
831 + if (!loc_as_list_contains_number(enumerator->asns, asn)) {
832 + DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
833 + return 1;
834 + }
835 + }
836 +
837 + // Skip if flags do not match
838 + if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
839 + DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
840 + return 1;
841 + }
842 +
843 + // Do not filter
844 + return 0;
845 +}
846 +
847 +static int __loc_database_enumerator_next_network(
848 + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
849 + // Return top element from the stack
850 + while (1) {
851 + *network = loc_network_list_pop_first(enumerator->stack);
852 +
853 + // Stack is empty
854 + if (!*network)
855 + break;
856 +
857 + // Throw away any networks by filter
858 + if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
859 + loc_network_unref(*network);
860 + *network = NULL;
861 + continue;
862 + }
863 +
864 + // Return result
865 + return 0;
866 + }
867
868 DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
869 enumerator->network_stack_depth);
870 @@ -1149,7 +1235,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
871 enumerator->db->network_nodes_v1 + node->offset;
872
873 // Add edges to stack
874 - r = loc_database_enumerator_stack_push_node(enumerator,
875 + int r = loc_database_enumerator_stack_push_node(enumerator,
876 be32toh(n->one), 1, node->depth + 1);
877
878 if (r)
879 @@ -1175,56 +1261,145 @@ LOC_EXPORT int loc_database_enumerator_next_network(
880 if (r)
881 return r;
882
883 - // Check if we are interested in this network
884 + // Return all networks when the filter is disabled
885 + if (!filter)
886 + return 0;
887
888 - // Skip if the family does not match
889 - if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
890 + // Check if we are interested in this network
891 + if (loc_database_enumerator_filter_network(enumerator, *network)) {
892 loc_network_unref(*network);
893 *network = NULL;
894
895 continue;
896 }
897
898 - // Skip if the country code does not match
899 - if (*enumerator->country_code &&
900 - !loc_network_match_country_code(*network, enumerator->country_code)) {
901 - loc_network_unref(*network);
902 - *network = NULL;
903 + return 0;
904 + }
905 + }
906
907 - continue;
908 - }
909 + // Reached the end of the search
910 + return 0;
911 +}
912
913 - // Skip if the ASN does not match
914 - if (enumerator->asn &&
915 - !loc_network_match_asn(*network, enumerator->asn)) {
916 - loc_network_unref(*network);
917 - *network = NULL;
918 +static int __loc_database_enumerator_next_network_flattened(
919 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
920 + // Fetch the next network
921 + int r = __loc_database_enumerator_next_network(enumerator, network, 1);
922 + if (r)
923 + return r;
924
925 - continue;
926 - }
927 + // End if we could not read another network
928 + if (!*network)
929 + return 0;
930
931 - // Skip if flags do not match
932 - if (enumerator->flags &&
933 - !loc_network_match_flag(*network, enumerator->flags)) {
934 - loc_network_unref(*network);
935 - *network = NULL;
936 + struct loc_network* subnet = NULL;
937 + struct loc_network_list* subnets;
938
939 - continue;
940 + // Create a list with all subnets
941 + r = loc_network_list_new(enumerator->ctx, &subnets);
942 + if (r)
943 + return r;
944 +
945 + // Search all subnets from the database
946 + while (1) {
947 + // Fetch the next network in line
948 + r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
949 + if (r) {
950 + loc_network_unref(subnet);
951 + loc_network_list_unref(subnets);
952 +
953 + return r;
954 + }
955 +
956 + // End if we did not receive another subnet
957 + if (!subnet)
958 + break;
959 +
960 + // Collect all subnets in a list
961 + if (loc_network_is_subnet(*network, subnet)) {
962 + r = loc_network_list_push(subnets, subnet);
963 + if (r) {
964 + loc_network_unref(subnet);
965 + loc_network_list_unref(subnets);
966 +
967 + return r;
968 }
969
970 - return 0;
971 + loc_network_unref(subnet);
972 + continue;
973 }
974 +
975 + // If this is not a subnet, we push it back onto the stack and break
976 + r = loc_network_list_push(enumerator->stack, subnet);
977 + if (r) {
978 + loc_network_unref(subnet);
979 + loc_network_list_unref(subnets);
980 +
981 + return r;
982 + }
983 +
984 + loc_network_unref(subnet);
985 + break;
986 }
987
988 - // Reached the end of the search
989 + DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
990 +
991 + // We can abort here if the network has no subnets
992 + if (loc_network_list_empty(subnets)) {
993 + loc_network_list_unref(subnets);
994 +
995 + return 0;
996 + }
997
998 - // Mark all nodes as non-visited
999 - for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
1000 - enumerator->networks_visited[i] = 0;
1001 + // If the network has any subnets, we will break it into smaller parts
1002 + // without the subnets.
1003 + struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
1004 + if (!excluded) {
1005 + loc_network_list_unref(subnets);
1006 + return -1;
1007 + }
1008 +
1009 + // Merge subnets onto the stack
1010 + r = loc_network_list_merge(enumerator->stack, subnets);
1011 + if (r) {
1012 + loc_network_list_unref(subnets);
1013 + loc_network_list_unref(excluded);
1014 +
1015 + return r;
1016 + }
1017 +
1018 + // Push excluded list onto the stack
1019 + r = loc_network_list_merge(enumerator->stack, excluded);
1020 + if (r) {
1021 + loc_network_list_unref(subnets);
1022 + loc_network_list_unref(excluded);
1023 +
1024 + return r;
1025 + }
1026 +
1027 + loc_network_list_unref(subnets);
1028 + loc_network_list_unref(excluded);
1029 +
1030 + // Replace network with the first one from the stack
1031 + loc_network_unref(*network);
1032 + *network = loc_network_list_pop_first(enumerator->stack);
1033
1034 return 0;
1035 }
1036
1037 +LOC_EXPORT int loc_database_enumerator_next_network(
1038 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
1039 + // Do not do anything if not in network mode
1040 + if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
1041 + return 0;
1042 +
1043 + // Flatten output?
1044 + if (enumerator->flatten)
1045 + return __loc_database_enumerator_next_network_flattened(enumerator, network);
1046 +
1047 + return __loc_database_enumerator_next_network(enumerator, network, 1);
1048 +}
1049 +
1050 LOC_EXPORT int loc_database_enumerator_next_country(
1051 struct loc_database_enumerator* enumerator, struct loc_country** country) {
1052 *country = NULL;
1053 diff --git a/src/libloc.sym b/src/libloc.sym
1054 index b8296eb..ee333f1 100644
1055 --- a/src/libloc.sym
1056 +++ b/src/libloc.sym
1057 @@ -37,6 +37,18 @@ global:
1058 loc_as_set_name;
1059 loc_as_unref;
1060
1061 + # AS List
1062 + loc_as_list_append;
1063 + loc_as_list_clear;
1064 + loc_as_list_contains;
1065 + loc_as_list_contains_number;
1066 + loc_as_list_empty;
1067 + loc_as_list_get;
1068 + loc_as_list_new;
1069 + loc_as_list_ref;
1070 + loc_as_list_size;
1071 + loc_as_list_unref;
1072 +
1073 # Country
1074 loc_country_cmp;
1075 loc_country_code_is_valid;
1076 @@ -49,6 +61,18 @@ global:
1077 loc_country_set_name;
1078 loc_country_unref;
1079
1080 + # Country List
1081 + loc_country_list_append;
1082 + loc_country_list_clear;
1083 + loc_country_list_contains;
1084 + loc_country_list_contains_code;
1085 + loc_country_list_empty;
1086 + loc_country_list_get;
1087 + loc_country_list_new;
1088 + loc_country_list_ref;
1089 + loc_country_list_size;
1090 + loc_country_list_unref;
1091 +
1092 # Database
1093 loc_database_add_as;
1094 loc_database_count_as;
1095 @@ -66,13 +90,15 @@ global:
1096 loc_database_verify;
1097
1098 # Database Enumerator
1099 + loc_database_enumerator_get_asns;
1100 + loc_database_enumerator_get_countries;
1101 loc_database_enumerator_new;
1102 loc_database_enumerator_next_as;
1103 loc_database_enumerator_next_country;
1104 loc_database_enumerator_next_network;
1105 loc_database_enumerator_ref;
1106 - loc_database_enumerator_set_asn;
1107 - loc_database_enumerator_set_country_code;
1108 + loc_database_enumerator_set_asns;
1109 + loc_database_enumerator_set_countries;
1110 loc_database_enumerator_set_family;
1111 loc_database_enumerator_set_flag;
1112 loc_database_enumerator_set_string;
1113 @@ -80,24 +106,48 @@ global:
1114
1115 # Network
1116 loc_network_address_family;
1117 + loc_network_cmp;
1118 + loc_network_exclude;
1119 + loc_network_exclude_list;
1120 loc_network_format_first_address;
1121 loc_network_format_last_address;
1122 loc_network_get_asn;
1123 loc_network_get_country_code;
1124 + loc_network_get_first_address;
1125 + loc_network_get_last_address;
1126 loc_network_has_flag;
1127 - loc_network_is_subnet_of;
1128 + loc_network_is_subnet;
1129 + loc_network_match_address;
1130 loc_network_match_asn;
1131 loc_network_match_country_code;
1132 loc_network_match_flag;
1133 loc_network_new;
1134 loc_network_new_from_string;
1135 + loc_network_overlaps;
1136 + loc_network_prefix;
1137 loc_network_ref;
1138 loc_network_set_asn;
1139 loc_network_set_country_code;
1140 loc_network_set_flag;
1141 loc_network_str;
1142 + loc_network_subnets;
1143 loc_network_unref;
1144
1145 + # Network List
1146 + loc_network_list_clear;
1147 + loc_network_list_contains;
1148 + loc_network_list_dump;
1149 + loc_network_list_empty;
1150 + loc_network_list_get;
1151 + loc_network_list_merge;
1152 + loc_network_list_new;
1153 + loc_network_list_pop;
1154 + loc_network_list_pop_first;
1155 + loc_network_list_push;
1156 + loc_network_list_ref;
1157 + loc_network_list_size;
1158 + loc_network_list_unref;
1159 +
1160 # Writer
1161 loc_writer_add_as;
1162 loc_writer_add_country;
1163 diff --git a/src/loc/as-list.h b/src/loc/as-list.h
1164 new file mode 100644
1165 index 0000000..7b5c4e8
1166 --- /dev/null
1167 +++ b/src/loc/as-list.h
1168 @@ -0,0 +1,41 @@
1169 +/*
1170 + libloc - A library to determine the location of someone on the Internet
1171 +
1172 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
1173 +
1174 + This library is free software; you can redistribute it and/or
1175 + modify it under the terms of the GNU Lesser General Public
1176 + License as published by the Free Software Foundation; either
1177 + version 2.1 of the License, or (at your option) any later version.
1178 +
1179 + This library is distributed in the hope that it will be useful,
1180 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1181 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1182 + Lesser General Public License for more details.
1183 +*/
1184 +
1185 +#ifndef LIBLOC_AS_LIST_H
1186 +#define LIBLOC_AS_LIST_H
1187 +
1188 +#include <loc/as.h>
1189 +#include <loc/libloc.h>
1190 +
1191 +struct loc_as_list;
1192 +
1193 +int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
1194 +struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
1195 +struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
1196 +
1197 +size_t loc_as_list_size(struct loc_as_list* list);
1198 +int loc_as_list_empty(struct loc_as_list* list);
1199 +void loc_as_list_clear(struct loc_as_list* list);
1200 +
1201 +struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
1202 +int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
1203 +
1204 +int loc_as_list_contains(
1205 + struct loc_as_list* list, struct loc_as* as);
1206 +int loc_as_list_contains_number(
1207 + struct loc_as_list* list, uint32_t number);
1208 +
1209 +#endif
1210 diff --git a/src/loc/country-list.h b/src/loc/country-list.h
1211 new file mode 100644
1212 index 0000000..a7f818a
1213 --- /dev/null
1214 +++ b/src/loc/country-list.h
1215 @@ -0,0 +1,43 @@
1216 +/*
1217 + libloc - A library to determine the location of someone on the Internet
1218 +
1219 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
1220 +
1221 + This library is free software; you can redistribute it and/or
1222 + modify it under the terms of the GNU Lesser General Public
1223 + License as published by the Free Software Foundation; either
1224 + version 2.1 of the License, or (at your option) any later version.
1225 +
1226 + This library is distributed in the hope that it will be useful,
1227 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1228 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1229 + Lesser General Public License for more details.
1230 +*/
1231 +
1232 +#ifndef LIBLOC_COUNTRY_LIST_H
1233 +#define LIBLOC_COUNTRY_LIST_H
1234 +
1235 +#include <stdlib.h>
1236 +
1237 +#include <loc/libloc.h>
1238 +#include <loc/country.h>
1239 +
1240 +struct loc_country_list;
1241 +
1242 +int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
1243 +struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
1244 +struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
1245 +
1246 +size_t loc_country_list_size(struct loc_country_list* list);
1247 +int loc_country_list_empty(struct loc_country_list* list);
1248 +void loc_country_list_clear(struct loc_country_list* list);
1249 +
1250 +struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
1251 +int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
1252 +
1253 +int loc_country_list_contains(
1254 + struct loc_country_list* list, struct loc_country* country);
1255 +int loc_country_list_contains_code(
1256 + struct loc_country_list* list, const char* code);
1257 +
1258 +#endif
1259 diff --git a/src/loc/database.h b/src/loc/database.h
1260 index 43173dd..70801f0 100644
1261 --- a/src/loc/database.h
1262 +++ b/src/loc/database.h
1263 @@ -25,6 +25,7 @@
1264 #include <loc/network.h>
1265 #include <loc/as.h>
1266 #include <loc/country.h>
1267 +#include <loc/country-list.h>
1268
1269 struct loc_database;
1270 int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
1271 @@ -55,15 +56,24 @@ enum loc_database_enumerator_mode {
1272 LOC_DB_ENUMERATE_COUNTRIES = 3,
1273 };
1274
1275 +enum loc_database_enumerator_flags {
1276 + LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
1277 +};
1278 +
1279 struct loc_database_enumerator;
1280 int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
1281 - struct loc_database* db, enum loc_database_enumerator_mode mode);
1282 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
1283 struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
1284 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
1285
1286 int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
1287 -int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
1288 -int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
1289 +struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
1290 +int loc_database_enumerator_set_countries(
1291 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
1292 +struct loc_as_list* loc_database_enumerator_get_asns(
1293 + struct loc_database_enumerator* enumerator);
1294 +int loc_database_enumerator_set_asns(
1295 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
1296 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
1297 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
1298 int loc_database_enumerator_next_as(
1299 diff --git a/src/loc/network-list.h b/src/loc/network-list.h
1300 new file mode 100644
1301 index 0000000..bee21c4
1302 --- /dev/null
1303 +++ b/src/loc/network-list.h
1304 @@ -0,0 +1,37 @@
1305 +/*
1306 + libloc - A library to determine the location of someone on the Internet
1307 +
1308 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
1309 +
1310 + This library is free software; you can redistribute it and/or
1311 + modify it under the terms of the GNU Lesser General Public
1312 + License as published by the Free Software Foundation; either
1313 + version 2.1 of the License, or (at your option) any later version.
1314 +
1315 + This library is distributed in the hope that it will be useful,
1316 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1317 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1318 + Lesser General Public License for more details.
1319 +*/
1320 +
1321 +#ifndef LIBLOC_NETWORK_LIST_H
1322 +#define LIBLOC_NETWORK_LIST_H
1323 +
1324 +#include <loc/network.h>
1325 +
1326 +struct loc_network_list;
1327 +int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
1328 +struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
1329 +struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
1330 +size_t loc_network_list_size(struct loc_network_list* list);
1331 +int loc_network_list_empty(struct loc_network_list* list);
1332 +void loc_network_list_clear(struct loc_network_list* list);
1333 +void loc_network_list_dump(struct loc_network_list* list);
1334 +struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
1335 +int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
1336 +struct loc_network* loc_network_list_pop(struct loc_network_list* list);
1337 +struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
1338 +int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
1339 +int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
1340 +
1341 +#endif
1342 diff --git a/src/loc/network.h b/src/loc/network.h
1343 index 70c3803..af3dafd 100644
1344 --- a/src/loc/network.h
1345 +++ b/src/loc/network.h
1346 @@ -21,6 +21,7 @@
1347
1348 #include <loc/libloc.h>
1349 #include <loc/format.h>
1350 +#include <loc/network-list.h>
1351
1352 enum loc_network_flags {
1353 LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
1354 @@ -37,8 +38,11 @@ struct loc_network* loc_network_ref(struct loc_network* network);
1355 struct loc_network* loc_network_unref(struct loc_network* network);
1356 char* loc_network_str(struct loc_network* network);
1357 int loc_network_address_family(struct loc_network* network);
1358 +unsigned int loc_network_prefix(struct loc_network* network);
1359
1360 +const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
1361 char* loc_network_format_first_address(struct loc_network* network);
1362 +const struct in6_addr* loc_network_get_last_address(struct loc_network* network);
1363 char* loc_network_format_last_address(struct loc_network* network);
1364 int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
1365
1366 @@ -54,7 +58,14 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
1367 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
1368 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
1369
1370 -int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
1371 +int loc_network_cmp(struct loc_network* self, struct loc_network* other);
1372 +int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
1373 +int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
1374 +int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
1375 +struct loc_network_list* loc_network_exclude(
1376 + struct loc_network* self, struct loc_network* other);
1377 +struct loc_network_list* loc_network_exclude_list(
1378 + struct loc_network* network, struct loc_network_list* list);
1379
1380 #ifdef LIBLOC_PRIVATE
1381
1382 diff --git a/src/network-list.c b/src/network-list.c
1383 new file mode 100644
1384 index 0000000..698d3ab
1385 --- /dev/null
1386 +++ b/src/network-list.c
1387 @@ -0,0 +1,299 @@
1388 +/*
1389 + libloc - A library to determine the location of someone on the Internet
1390 +
1391 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
1392 +
1393 + This library is free software; you can redistribute it and/or
1394 + modify it under the terms of the GNU Lesser General Public
1395 + License as published by the Free Software Foundation; either
1396 + version 2.1 of the License, or (at your option) any later version.
1397 +
1398 + This library is distributed in the hope that it will be useful,
1399 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1400 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1401 + Lesser General Public License for more details.
1402 +*/
1403 +
1404 +#include <errno.h>
1405 +#include <stdlib.h>
1406 +#include <time.h>
1407 +
1408 +#include <loc/libloc.h>
1409 +#include <loc/network.h>
1410 +#include <loc/private.h>
1411 +
1412 +struct loc_network_list {
1413 + struct loc_ctx* ctx;
1414 + int refcount;
1415 +
1416 + struct loc_network** elements;
1417 + size_t elements_size;
1418 +
1419 + size_t size;
1420 +};
1421 +
1422 +static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
1423 + DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
1424 + list, size, list->elements_size + size);
1425 +
1426 + struct loc_network** elements = reallocarray(list->elements,
1427 + list->elements_size + size, sizeof(*list->elements));
1428 + if (!elements)
1429 + return -errno;
1430 +
1431 + list->elements = elements;
1432 + list->elements_size += size;
1433 +
1434 + return 0;
1435 +}
1436 +
1437 +LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
1438 + struct loc_network_list** list) {
1439 + struct loc_network_list* l = calloc(1, sizeof(*l));
1440 + if (!l)
1441 + return -ENOMEM;
1442 +
1443 + l->ctx = loc_ref(ctx);
1444 + l->refcount = 1;
1445 +
1446 + DEBUG(l->ctx, "Network list allocated at %p\n", l);
1447 + *list = l;
1448 + return 0;
1449 +}
1450 +
1451 +LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
1452 + list->refcount++;
1453 +
1454 + return list;
1455 +}
1456 +
1457 +static void loc_network_list_free(struct loc_network_list* list) {
1458 + DEBUG(list->ctx, "Releasing network list at %p\n", list);
1459 +
1460 + // Remove all content
1461 + loc_network_list_clear(list);
1462 +
1463 + loc_unref(list->ctx);
1464 + free(list);
1465 +}
1466 +
1467 +LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
1468 + if (!list)
1469 + return NULL;
1470 +
1471 + if (--list->refcount > 0)
1472 + return list;
1473 +
1474 + loc_network_list_free(list);
1475 + return NULL;
1476 +}
1477 +
1478 +LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
1479 + return list->size;
1480 +}
1481 +
1482 +LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
1483 + return list->size == 0;
1484 +}
1485 +
1486 +LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
1487 + if (!list->elements)
1488 + return;
1489 +
1490 + for (unsigned int i = 0; i < list->size; i++)
1491 + loc_network_unref(list->elements[i]);
1492 +
1493 + free(list->elements);
1494 + list->elements = NULL;
1495 + list->elements_size = 0;
1496 +
1497 + list->size = 0;
1498 +}
1499 +
1500 +LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
1501 + struct loc_network* network;
1502 + char* s;
1503 +
1504 + for (unsigned int i = 0; i < list->size; i++) {
1505 + network = list->elements[i];
1506 +
1507 + s = loc_network_str(network);
1508 +
1509 + INFO(list->ctx, "%4d: %s\n", i, s);
1510 + free(s);
1511 + }
1512 +}
1513 +
1514 +LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
1515 + // Check index
1516 + if (index >= list->size)
1517 + return NULL;
1518 +
1519 + return loc_network_ref(list->elements[index]);
1520 +}
1521 +
1522 +static off_t loc_network_list_find(struct loc_network_list* list,
1523 + struct loc_network* network, int* found) {
1524 + // Insert at the beginning for an empty list
1525 + if (loc_network_list_empty(list))
1526 + return 0;
1527 +
1528 + off_t lo = 0;
1529 + off_t hi = list->size - 1;
1530 + int result;
1531 +
1532 + // Since we are working on an ordered list, there is often a good chance that
1533 + // the network we are looking for is at the end or has to go to the end.
1534 + if (hi >= 0) {
1535 + result = loc_network_cmp(network, list->elements[hi]);
1536 +
1537 + // Match, so we are done
1538 + if (result == 0) {
1539 + *found = 1;
1540 +
1541 + return hi;
1542 +
1543 + // This needs to be added after the last one
1544 + } else if (result > 0) {
1545 + *found = 0;
1546 +
1547 + return hi + 1;
1548 + }
1549 + }
1550 +
1551 +#ifdef ENABLE_DEBUG
1552 + // Save start time
1553 + clock_t start = clock();
1554 +#endif
1555 +
1556 + off_t i = 0;
1557 +
1558 + while (lo <= hi) {
1559 + i = (lo + hi) / 2;
1560 +
1561 + // Check if this is a match
1562 + result = loc_network_cmp(network, list->elements[i]);
1563 +
1564 + if (result == 0) {
1565 + *found = 1;
1566 +
1567 +#ifdef ENABLE_DEBUG
1568 + clock_t end = clock();
1569 +
1570 + // Log how fast this has been
1571 + DEBUG(list->ctx, "Found network in %.4fms at %jd\n",
1572 + (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
1573 +#endif
1574 +
1575 + return i;
1576 + }
1577 +
1578 + if (result > 0) {
1579 + lo = i + 1;
1580 + i++;
1581 + } else {
1582 + hi = i - 1;
1583 + }
1584 + }
1585 +
1586 + *found = 0;
1587 +
1588 +#ifdef ENABLE_DEBUG
1589 + clock_t end = clock();
1590 +
1591 + // Log how fast this has been
1592 + DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n",
1593 + (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
1594 +#endif
1595 +
1596 + return i;
1597 +}
1598 +
1599 +LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
1600 + int found = 0;
1601 +
1602 + off_t index = loc_network_list_find(list, network, &found);
1603 +
1604 + // The network has been found on the list. Nothing to do.
1605 + if (found)
1606 + return 0;
1607 +
1608 + DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n",
1609 + list, network, (intmax_t)index);
1610 +
1611 + // Check if we have space left
1612 + if (list->size >= list->elements_size) {
1613 + int r = loc_network_list_grow(list, 64);
1614 + if (r)
1615 + return r;
1616 + }
1617 +
1618 + // The list is now larger
1619 + list->size++;
1620 +
1621 + // Move all elements out of the way
1622 + for (unsigned int i = list->size - 1; i > index; i--)
1623 + list->elements[i] = list->elements[i - 1];
1624 +
1625 + // Add the new element at the right place
1626 + list->elements[index] = loc_network_ref(network);
1627 +
1628 + return 0;
1629 +}
1630 +
1631 +LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
1632 + // Return nothing when empty
1633 + if (loc_network_list_empty(list)) {
1634 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
1635 + return NULL;
1636 + }
1637 +
1638 + struct loc_network* network = list->elements[--list->size];
1639 +
1640 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
1641 +
1642 + return network;
1643 +}
1644 +
1645 +LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
1646 + // Return nothing when empty
1647 + if (loc_network_list_empty(list)) {
1648 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
1649 + return NULL;
1650 + }
1651 +
1652 + struct loc_network* network = list->elements[0];
1653 +
1654 + // Move all elements to the top of the stack
1655 + for (unsigned int i = 0; i < list->size - 1; i++) {
1656 + list->elements[i] = list->elements[i+1];
1657 + }
1658 +
1659 + // The list is shorter now
1660 + --list->size;
1661 +
1662 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
1663 +
1664 + return network;
1665 +}
1666 +
1667 +LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
1668 + int found = 0;
1669 +
1670 + loc_network_list_find(list, network, &found);
1671 +
1672 + return found;
1673 +}
1674 +
1675 +LOC_EXPORT int loc_network_list_merge(
1676 + struct loc_network_list* self, struct loc_network_list* other) {
1677 + int r;
1678 +
1679 + for (unsigned int i = 0; i < other->size; i++) {
1680 + r = loc_network_list_push(self, other->elements[i]);
1681 + if (r)
1682 + return r;
1683 + }
1684 +
1685 + return 0;
1686 +}
1687 diff --git a/src/network.c b/src/network.c
1688 index 366caa2..a6b679c 100644
1689 --- a/src/network.c
1690 +++ b/src/network.c
1691 @@ -29,6 +29,7 @@
1692 #include <loc/compat.h>
1693 #include <loc/country.h>
1694 #include <loc/network.h>
1695 +#include <loc/network-list.h>
1696 #include <loc/private.h>
1697
1698 struct loc_network {
1699 @@ -97,6 +98,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
1700 return a;
1701 }
1702
1703 +static struct in6_addr address_increment(const struct in6_addr* address) {
1704 + struct in6_addr a = *address;
1705 +
1706 + for (int octet = 15; octet >= 0; octet--) {
1707 + if (a.s6_addr[octet] < 255) {
1708 + a.s6_addr[octet]++;
1709 + break;
1710 + } else {
1711 + a.s6_addr[octet] = 0;
1712 + }
1713 + }
1714 +
1715 + return a;
1716 +}
1717 +
1718 LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
1719 struct in6_addr* address, unsigned int prefix) {
1720 // Address cannot be unspecified
1721 @@ -160,9 +176,11 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
1722 LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
1723 const char* address_string) {
1724 struct in6_addr first_address;
1725 - unsigned int prefix = 0;
1726 char* prefix_string;
1727 - int r = 1;
1728 + unsigned int prefix = 128;
1729 + int r = -EINVAL;
1730 +
1731 + DEBUG(ctx, "Attempting to parse network %s\n", address_string);
1732
1733 // Make a copy of the string to work on it
1734 char* buffer = strdup(address_string);
1735 @@ -171,29 +189,40 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1736 // Split address and prefix
1737 address_string = strsep(&prefix_string, "/");
1738
1739 - // Did we find a prefix?
1740 + DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
1741 +
1742 + // Parse the address
1743 + r = loc_parse_address(ctx, address_string, &first_address);
1744 + if (r) {
1745 + DEBUG(ctx, "The address could not be parsed\n");
1746 + goto FAIL;
1747 + }
1748 +
1749 + // If a prefix was given, we will try to parse it
1750 if (prefix_string) {
1751 // Convert prefix to integer
1752 prefix = strtol(prefix_string, NULL, 10);
1753
1754 - if (prefix) {
1755 - // Parse the address
1756 - r = loc_parse_address(ctx, address_string, &first_address);
1757 -
1758 - // Map the prefix to IPv6 if needed
1759 - if (IN6_IS_ADDR_V4MAPPED(&first_address))
1760 - prefix += 96;
1761 + if (!prefix) {
1762 + DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
1763 + goto FAIL;
1764 }
1765 +
1766 + // Map the prefix to IPv6 if needed
1767 + if (IN6_IS_ADDR_V4MAPPED(&first_address))
1768 + prefix += 96;
1769 }
1770
1771 +FAIL:
1772 // Free temporary buffer
1773 free(buffer);
1774
1775 - if (r == 0) {
1776 - r = loc_network_new(ctx, network, &first_address, prefix);
1777 - }
1778 + // Exit if the parsing was unsuccessful
1779 + if (r)
1780 + return r;
1781
1782 - return r;
1783 + // Create a new network
1784 + return loc_network_new(ctx, network, &first_address, prefix);
1785 }
1786
1787 LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
1788 @@ -281,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) {
1789 return network->family;
1790 }
1791
1792 +LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) {
1793 + switch (network->family) {
1794 + case AF_INET6:
1795 + return network->prefix;
1796 +
1797 + case AF_INET:
1798 + return network->prefix - 96;
1799 + }
1800 +
1801 + return 0;
1802 +}
1803 +
1804 static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) {
1805 const size_t length = INET6_ADDRSTRLEN;
1806
1807 @@ -314,10 +355,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc
1808 return string;
1809 }
1810
1811 +LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) {
1812 + return &network->first_address;
1813 +}
1814 +
1815 LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) {
1816 return loc_network_format_address(network, &network->first_address);
1817 }
1818
1819 +LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) {
1820 + return &network->last_address;
1821 +}
1822 +
1823 LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
1824 return loc_network_format_address(network, &network->last_address);
1825 }
1826 @@ -325,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
1827 LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) {
1828 // Address must be larger than the start address
1829 if (in6_addr_cmp(&network->first_address, address) > 0)
1830 - return 1;
1831 + return 0;
1832
1833 // Address must be smaller than the last address
1834 if (in6_addr_cmp(&network->last_address, address) < 0)
1835 - return 1;
1836 + return 0;
1837
1838 // The address is inside this network
1839 - return 0;
1840 + return 1;
1841 }
1842
1843 LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) {
1844 @@ -392,20 +441,310 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
1845 return loc_network_has_flag(network, flag);
1846 }
1847
1848 -LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
1849 +LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
1850 + // Compare address
1851 + int r = in6_addr_cmp(&self->first_address, &other->first_address);
1852 + if (r)
1853 + return r;
1854 +
1855 + // Compare prefix
1856 + if (self->prefix > other->prefix)
1857 + return 1;
1858 + else if (self->prefix < other->prefix)
1859 + return -1;
1860 +
1861 + // Both networks are equal
1862 + return 0;
1863 +}
1864 +
1865 +LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
1866 + // Either of the start addresses must be in the other subnet
1867 + if (loc_network_match_address(self, &other->first_address))
1868 + return 1;
1869 +
1870 + if (loc_network_match_address(other, &self->first_address))
1871 + return 1;
1872 +
1873 + // Or either of the end addresses is in the other subnet
1874 + if (loc_network_match_address(self, &other->last_address))
1875 + return 1;
1876 +
1877 + if (loc_network_match_address(other, &self->last_address))
1878 + return 1;
1879 +
1880 + return 0;
1881 +}
1882 +
1883 +LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
1884 + // The prefix must be smaller (this avoids the more complex comparisons later)
1885 + if (self->prefix > other->prefix)
1886 + return 0;
1887 +
1888 // If the start address of the other network is smaller than this network,
1889 // it cannot be a subnet.
1890 - if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
1891 + if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
1892 return 0;
1893
1894 // If the end address of the other network is greater than this network,
1895 // it cannot be a subnet.
1896 - if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
1897 + if (in6_addr_cmp(&self->last_address, &other->last_address) < 0)
1898 return 0;
1899
1900 return 1;
1901 }
1902
1903 +LOC_EXPORT int loc_network_subnets(struct loc_network* network,
1904 + struct loc_network** subnet1, struct loc_network** subnet2) {
1905 + int r;
1906 + *subnet1 = NULL;
1907 + *subnet2 = NULL;
1908 +
1909 + // New prefix length
1910 + unsigned int prefix = network->prefix + 1;
1911 +
1912 + // Check if the new prefix is valid
1913 + if (valid_prefix(&network->first_address, prefix))
1914 + return -1;
1915 +
1916 + // Create the first half of the network
1917 + r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix);
1918 + if (r)
1919 + return r;
1920 +
1921 + // The next subnet starts after the first one
1922 + struct in6_addr first_address = address_increment(&(*subnet1)->last_address);
1923 +
1924 + // Create the second half of the network
1925 + r = loc_network_new(network->ctx, subnet2, &first_address, prefix);
1926 + if (r)
1927 + return r;
1928 +
1929 + // Copy country code
1930 + const char* country_code = loc_network_get_country_code(network);
1931 + if (country_code) {
1932 + loc_network_set_country_code(*subnet1, country_code);
1933 + loc_network_set_country_code(*subnet2, country_code);
1934 + }
1935 +
1936 + // Copy ASN
1937 + uint32_t asn = loc_network_get_asn(network);
1938 + if (asn) {
1939 + loc_network_set_asn(*subnet1, asn);
1940 + loc_network_set_asn(*subnet2, asn);
1941 + }
1942 +
1943 + // Copy flags
1944 + loc_network_set_flag(*subnet1, network->flags);
1945 + loc_network_set_flag(*subnet2, network->flags);
1946 +
1947 + return 0;
1948 +}
1949 +
1950 +static int __loc_network_exclude(struct loc_network* network,
1951 + struct loc_network* other, struct loc_network_list* list) {
1952 + struct loc_network* subnet1 = NULL;
1953 + struct loc_network* subnet2 = NULL;
1954 +
1955 + int r = loc_network_subnets(network, &subnet1, &subnet2);
1956 + if (r)
1957 + goto ERROR;
1958 +
1959 + if (loc_network_cmp(other, subnet1) == 0) {
1960 + r = loc_network_list_push(list, subnet2);
1961 + if (r)
1962 + goto ERROR;
1963 +
1964 + } else if (loc_network_cmp(other, subnet2) == 0) {
1965 + r = loc_network_list_push(list, subnet1);
1966 + if (r)
1967 + goto ERROR;
1968 +
1969 + } else if (loc_network_is_subnet(subnet1, other)) {
1970 + r = loc_network_list_push(list, subnet2);
1971 + if (r)
1972 + goto ERROR;
1973 +
1974 + r = __loc_network_exclude(subnet1, other, list);
1975 + if (r)
1976 + goto ERROR;
1977 +
1978 + } else if (loc_network_is_subnet(subnet2, other)) {
1979 + r = loc_network_list_push(list, subnet1);
1980 + if (r)
1981 + goto ERROR;
1982 +
1983 + r = __loc_network_exclude(subnet2, other, list);
1984 + if (r)
1985 + goto ERROR;
1986 +
1987 + } else {
1988 + ERROR(network->ctx, "We should never get here\n");
1989 + r = 1;
1990 + goto ERROR;
1991 + }
1992 +
1993 +ERROR:
1994 + if (subnet1)
1995 + loc_network_unref(subnet1);
1996 +
1997 + if (subnet2)
1998 + loc_network_unref(subnet2);
1999 +
2000 + return r;
2001 +}
2002 +
2003 +static int __loc_network_exclude_to_list(struct loc_network* self,
2004 + struct loc_network* other, struct loc_network_list* list) {
2005 + // Other must be a subnet of self
2006 + if (!loc_network_is_subnet(self, other)) {
2007 + DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
2008 +
2009 + // Exit silently
2010 + return 0;
2011 + }
2012 +
2013 + // We cannot perform this operation if both networks equal
2014 + if (loc_network_cmp(self, other) == 0) {
2015 + DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
2016 +
2017 + // Exit silently
2018 + return 0;
2019 + }
2020 +
2021 + return __loc_network_exclude(self, other, list);
2022 +}
2023 +
2024 +LOC_EXPORT struct loc_network_list* loc_network_exclude(
2025 + struct loc_network* self, struct loc_network* other) {
2026 + struct loc_network_list* list;
2027 +
2028 +#ifdef ENABLE_DEBUG
2029 + char* n1 = loc_network_str(self);
2030 + char* n2 = loc_network_str(other);
2031 +
2032 + DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
2033 +
2034 + free(n1);
2035 + free(n2);
2036 +#endif
2037 +
2038 + // Create a new list with the result
2039 + int r = loc_network_list_new(self->ctx, &list);
2040 + if (r) {
2041 + ERROR(self->ctx, "Could not create network list: %d\n", r);
2042 +
2043 + return NULL;
2044 + }
2045 +
2046 + r = __loc_network_exclude_to_list(self, other, list);
2047 + if (r) {
2048 + loc_network_list_unref(list);
2049 +
2050 + return NULL;
2051 + }
2052 +
2053 + // Return the result
2054 + return list;
2055 +}
2056 +
2057 +LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
2058 + struct loc_network* network, struct loc_network_list* list) {
2059 + struct loc_network_list* to_check;
2060 +
2061 + // Create a new list with all networks to look at
2062 + int r = loc_network_list_new(network->ctx, &to_check);
2063 + if (r)
2064 + return NULL;
2065 +
2066 + struct loc_network* subnet = NULL;
2067 + struct loc_network_list* subnets = NULL;
2068 +
2069 + for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
2070 + subnet = loc_network_list_get(list, i);
2071 +
2072 + // Find all excluded networks
2073 + if (!loc_network_list_contains(to_check, subnet)) {
2074 + r = __loc_network_exclude_to_list(network, subnet, to_check);
2075 + if (r) {
2076 + loc_network_list_unref(to_check);
2077 + loc_network_unref(subnet);
2078 +
2079 + return NULL;
2080 + }
2081 + }
2082 +
2083 + // Cleanup
2084 + loc_network_unref(subnet);
2085 + }
2086 +
2087 + r = loc_network_list_new(network->ctx, &subnets);
2088 + if (r) {
2089 + loc_network_list_unref(to_check);
2090 + return NULL;
2091 + }
2092 +
2093 + off_t smallest_subnet = 0;
2094 +
2095 + while (!loc_network_list_empty(to_check)) {
2096 + struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check);
2097 +
2098 + // Check whether the subnet to check is part of the input list
2099 + if (loc_network_list_contains(list, subnet_to_check)) {
2100 + loc_network_unref(subnet_to_check);
2101 + continue;
2102 + }
2103 +
2104 + // Marks whether this subnet passed all checks
2105 + int passed = 1;
2106 +
2107 + for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) {
2108 + subnet = loc_network_list_get(list, i);
2109 +
2110 + // Drop this subnet if is a subnet of another subnet
2111 + if (loc_network_is_subnet(subnet, subnet_to_check)) {
2112 + passed = 0;
2113 + loc_network_unref(subnet);
2114 + break;
2115 + }
2116 +
2117 + // Break it down if it overlaps
2118 + if (loc_network_overlaps(subnet, subnet_to_check)) {
2119 + passed = 0;
2120 +
2121 + __loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
2122 +
2123 + loc_network_unref(subnet);
2124 + break;
2125 + }
2126 +
2127 + // If the subnet is strictly greater, we do not need to continue the search
2128 + r = loc_network_cmp(subnet, subnet_to_check);
2129 + if (r > 0) {
2130 + loc_network_unref(subnet);
2131 + break;
2132 +
2133 + // If it is strictly smaller, we can continue the search from here next
2134 + // time because all networks that are to be checked can only be larger
2135 + // than this one.
2136 + } else if (r < 0) {
2137 + smallest_subnet = i;
2138 + }
2139 +
2140 + loc_network_unref(subnet);
2141 + }
2142 +
2143 + if (passed) {
2144 + r = loc_network_list_push(subnets, subnet_to_check);
2145 + }
2146 +
2147 + loc_network_unref(subnet_to_check);
2148 + }
2149 +
2150 + loc_network_list_unref(to_check);
2151 +
2152 + return subnets;
2153 +}
2154 +
2155 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
2156 // Add country code
2157 loc_country_code_copy(dbobj->country_code, network->country_code);
2158 @@ -474,7 +813,7 @@ struct loc_network_tree_node {
2159 struct loc_network* network;
2160 };
2161
2162 -LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
2163 +int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
2164 struct loc_network_tree* t = calloc(1, sizeof(*t));
2165 if (!t)
2166 return -ENOMEM;
2167 @@ -494,7 +833,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
2168 return 0;
2169 }
2170
2171 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
2172 +struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
2173 return loc_network_tree_node_ref(tree->root);
2174 }
2175
2176 @@ -566,7 +905,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
2177 return 0;
2178 }
2179
2180 -LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
2181 +int loc_network_tree_walk(struct loc_network_tree* tree,
2182 int(*filter_callback)(struct loc_network* network, void* data),
2183 int(*callback)(struct loc_network* network, void* data), void* data) {
2184 return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
2185 @@ -581,7 +920,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
2186 free(tree);
2187 }
2188
2189 -LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
2190 +struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
2191 if (--tree->refcount > 0)
2192 return tree;
2193
2194 @@ -602,13 +941,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
2195 return 0;
2196 }
2197
2198 -LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
2199 +int loc_network_tree_dump(struct loc_network_tree* tree) {
2200 DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
2201
2202 return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
2203 }
2204
2205 -LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
2206 +int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
2207 DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
2208
2209 struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
2210 @@ -639,7 +978,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
2211 return 0;
2212 }
2213
2214 -LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
2215 +size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
2216 size_t counter = 0;
2217
2218 int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
2219 @@ -661,11 +1000,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
2220 return counter;
2221 }
2222
2223 -LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
2224 +size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
2225 return __loc_network_tree_count_nodes(tree->root);
2226 }
2227
2228 -LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
2229 +int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
2230 struct loc_network_tree_node* n = calloc(1, sizeof(*n));
2231 if (!n)
2232 return -ENOMEM;
2233 @@ -680,7 +1019,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
2234 return 0;
2235 }
2236
2237 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
2238 +struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
2239 if (node)
2240 node->refcount++;
2241
2242 @@ -703,7 +1042,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
2243 free(node);
2244 }
2245
2246 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
2247 +struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
2248 if (!node)
2249 return NULL;
2250
2251 @@ -714,7 +1053,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
2252 return NULL;
2253 }
2254
2255 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
2256 +struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
2257 if (index == 0)
2258 node = node->zero;
2259 else
2260 @@ -726,10 +1065,10 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
2261 return loc_network_tree_node_ref(node);
2262 }
2263
2264 -LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
2265 +int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
2266 return (!!node->network);
2267 }
2268
2269 -LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2270 +struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2271 return loc_network_ref(node->network);
2272 }
2273 diff --git a/src/perl/Location.xs b/src/perl/Location.xs
2274 index dcf3f0d..b7676d2 100644
2275 --- a/src/perl/Location.xs
2276 +++ b/src/perl/Location.xs
2277 @@ -125,7 +125,7 @@ database_countries(db)
2278 PPCODE:
2279 // Create Database enumerator
2280 struct loc_database_enumerator* enumerator;
2281 - int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
2282 + int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
2283
2284 if (err) {
2285 croak("Could not create a database enumerator\n");
2286 diff --git a/src/python/database.c b/src/python/database.c
2287 index 1013a58..0aa03cc 100644
2288 --- a/src/python/database.c
2289 +++ b/src/python/database.c
2290 @@ -17,6 +17,8 @@
2291 #include <Python.h>
2292
2293 #include <loc/libloc.h>
2294 +#include <loc/as.h>
2295 +#include <loc/as-list.h>
2296 #include <loc/database.h>
2297
2298 #include "locationmodule.h"
2299 @@ -207,10 +209,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
2300 return (PyObject*)self;
2301 }
2302
2303 -static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
2304 +static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
2305 struct loc_database_enumerator* enumerator;
2306
2307 - int r = loc_database_enumerator_new(&enumerator, self->db, what);
2308 + int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
2309 if (r) {
2310 PyErr_SetFromErrno(PyExc_SystemError);
2311 return NULL;
2312 @@ -223,7 +225,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
2313 }
2314
2315 static PyObject* Database_ases(DatabaseObject* self) {
2316 - return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
2317 + return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
2318 }
2319
2320 static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2321 @@ -234,7 +236,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2322
2323 struct loc_database_enumerator* enumerator;
2324
2325 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
2326 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
2327 if (r) {
2328 PyErr_SetFromErrno(PyExc_SystemError);
2329 return NULL;
2330 @@ -250,44 +252,142 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2331 }
2332
2333 static PyObject* Database_networks(DatabaseObject* self) {
2334 - return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
2335 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
2336 +}
2337 +
2338 +static PyObject* Database_networks_flattened(DatabaseObject *self) {
2339 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
2340 }
2341
2342 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
2343 - char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
2344 - const char* country_code = NULL;
2345 - unsigned int asn = 0;
2346 + char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
2347 + PyObject* country_codes = NULL;
2348 + PyObject* asn_list = NULL;
2349 int flags = 0;
2350 int family = 0;
2351 + int flatten = 0;
2352
2353 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
2354 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
2355 + &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
2356 return NULL;
2357
2358 struct loc_database_enumerator* enumerator;
2359 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
2360 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
2361 + (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
2362 if (r) {
2363 PyErr_SetFromErrno(PyExc_SystemError);
2364 return NULL;
2365 }
2366
2367 // Set country code we are searching for
2368 - if (country_code) {
2369 - r = loc_database_enumerator_set_country_code(enumerator, country_code);
2370 + if (country_codes) {
2371 + struct loc_country_list* countries;
2372 + r = loc_country_list_new(loc_ctx, &countries);
2373 + if (r) {
2374 + PyErr_SetString(PyExc_SystemError, "Could not create country list");
2375 + return NULL;
2376 + }
2377 +
2378 + for (int i = 0; i < PyList_Size(country_codes); i++) {
2379 + PyObject* item = PyList_GetItem(country_codes, i);
2380 +
2381 + if (!PyUnicode_Check(item)) {
2382 + PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
2383 + loc_country_list_unref(countries);
2384 + return NULL;
2385 + }
2386 +
2387 + const char* country_code = PyUnicode_AsUTF8(item);
2388 +
2389 + struct loc_country* country;
2390 + r = loc_country_new(loc_ctx, &country, country_code);
2391 + if (r) {
2392 + if (r == -EINVAL) {
2393 + PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
2394 + } else {
2395 + PyErr_SetString(PyExc_SystemError, "Could not create country");
2396 + }
2397 +
2398 + loc_country_list_unref(countries);
2399 + return NULL;
2400 + }
2401 +
2402 + // Append it to the list
2403 + r = loc_country_list_append(countries, country);
2404 + if (r) {
2405 + PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
2406 +
2407 + loc_country_list_unref(countries);
2408 + loc_country_unref(country);
2409 + return NULL;
2410 + }
2411 +
2412 + loc_country_unref(country);
2413 + }
2414
2415 + r = loc_database_enumerator_set_countries(enumerator, countries);
2416 if (r) {
2417 PyErr_SetFromErrno(PyExc_SystemError);
2418 +
2419 + loc_country_list_unref(countries);
2420 return NULL;
2421 }
2422 +
2423 + loc_country_list_unref(countries);
2424 }
2425
2426 // Set the ASN we are searching for
2427 - if (asn) {
2428 - r = loc_database_enumerator_set_asn(enumerator, asn);
2429 + if (asn_list) {
2430 + struct loc_as_list* asns;
2431 + r = loc_as_list_new(loc_ctx, &asns);
2432 + if (r) {
2433 + PyErr_SetString(PyExc_SystemError, "Could not create AS list");
2434 + return NULL;
2435 + }
2436 +
2437 + for (int i = 0; i < PyList_Size(asn_list); i++) {
2438 + PyObject* item = PyList_GetItem(asn_list, i);
2439 +
2440 + if (!PyLong_Check(item)) {
2441 + PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
2442
2443 + loc_as_list_unref(asns);
2444 + return NULL;
2445 + }
2446 +
2447 + unsigned long number = PyLong_AsLong(item);
2448 +
2449 + struct loc_as* as;
2450 + r = loc_as_new(loc_ctx, &as, number);
2451 + if (r) {
2452 + PyErr_SetString(PyExc_SystemError, "Could not create AS");
2453 +
2454 + loc_as_list_unref(asns);
2455 + loc_as_unref(as);
2456 + return NULL;
2457 + }
2458 +
2459 + r = loc_as_list_append(asns, as);
2460 + if (r) {
2461 + PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
2462 +
2463 + loc_as_list_unref(asns);
2464 + loc_as_unref(as);
2465 + return NULL;
2466 + }
2467 +
2468 + loc_as_unref(as);
2469 + }
2470 +
2471 + r = loc_database_enumerator_set_asns(enumerator, asns);
2472 if (r) {
2473 PyErr_SetFromErrno(PyExc_SystemError);
2474 +
2475 + loc_as_list_unref(asns);
2476 return NULL;
2477 }
2478 +
2479 + loc_as_list_unref(asns);
2480 }
2481
2482 // Set the flags we are searching for
2483 @@ -317,7 +417,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
2484 }
2485
2486 static PyObject* Database_countries(DatabaseObject* self) {
2487 - return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
2488 + return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
2489 }
2490
2491 static struct PyMethodDef Database_methods[] = {
2492 @@ -403,6 +503,13 @@ static struct PyGetSetDef Database_getsetters[] = {
2493 NULL,
2494 NULL,
2495 },
2496 + {
2497 + "networks_flattened",
2498 + (getter)Database_networks_flattened,
2499 + NULL,
2500 + NULL,
2501 + NULL,
2502 + },
2503 {
2504 "vendor",
2505 (getter)Database_get_vendor,
2506 diff --git a/src/python/downloader.py b/src/python/downloader.py
2507 index 87bbb68..05f7872 100644
2508 --- a/src/python/downloader.py
2509 +++ b/src/python/downloader.py
2510 @@ -119,8 +119,8 @@ class Downloader(object):
2511
2512 headers = {}
2513 if timestamp:
2514 - headers["If-Modified-Since"] = timestamp.strftime(
2515 - "%a, %d %b %Y %H:%M:%S GMT",
2516 + headers["If-Modified-Since"] = time.strftime(
2517 + "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
2518 )
2519
2520 t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
2521 @@ -195,7 +195,7 @@ class Downloader(object):
2522 db = Database(f.name)
2523
2524 # Database is not recent
2525 - if timestamp and db.created_at < timestamp.timestamp():
2526 + if timestamp and db.created_at < timestamp:
2527 return False
2528
2529 log.info("Downloaded new database from %s" % (time.strftime(
2530 diff --git a/src/python/export.py b/src/python/export.py
2531 index d15c6f0..f0eae26 100644
2532 --- a/src/python/export.py
2533 +++ b/src/python/export.py
2534 @@ -29,7 +29,7 @@ import _location
2535 log = logging.getLogger("location.export")
2536 log.propagate = 1
2537
2538 -flags = {
2539 +FLAGS = {
2540 _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
2541 _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
2542 _location.NETWORK_FLAG_ANYCAST : "A3",
2543 @@ -39,11 +39,8 @@ class OutputWriter(object):
2544 suffix = "networks"
2545 mode = "w"
2546
2547 - def __init__(self, f, prefix=None, flatten=True):
2548 - self.f, self.prefix, self.flatten = f, prefix, flatten
2549 -
2550 - # The previously written network
2551 - self._last_network = None
2552 + def __init__(self, f, prefix=None):
2553 + self.f, self.prefix = f, prefix
2554
2555 # Immediately write the header
2556 self._write_header()
2557 @@ -60,18 +57,6 @@ class OutputWriter(object):
2558 def __repr__(self):
2559 return "<%s f=%s>" % (self.__class__.__name__, self.f)
2560
2561 - def _flatten(self, network):
2562 - """
2563 - Checks if the given network needs to be written to file,
2564 - or if it is a subnet of the previously written network.
2565 - """
2566 - if self._last_network and network.is_subnet_of(self._last_network):
2567 - return True
2568 -
2569 - # Remember this network for the next call
2570 - self._last_network = network
2571 - return False
2572 -
2573 def _write_header(self):
2574 """
2575 The header of the file
2576 @@ -84,16 +69,8 @@ class OutputWriter(object):
2577 """
2578 pass
2579
2580 - def _write_network(self, network):
2581 - self.f.write("%s\n" % network)
2582 -
2583 def write(self, network):
2584 - if self.flatten and self._flatten(network):
2585 - log.debug("Skipping writing network %s" % network)
2586 - return
2587 -
2588 - # Write the network to file
2589 - self._write_network(network)
2590 + self.f.write("%s\n" % network)
2591
2592 def finish(self):
2593 """
2594 @@ -114,7 +91,7 @@ class IpsetOutputWriter(OutputWriter):
2595 def _write_header(self):
2596 self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
2597
2598 - def _write_network(self, network):
2599 + def write(self, network):
2600 self.f.write("add %s %s\n" % (self.prefix, network))
2601
2602
2603 @@ -130,7 +107,7 @@ class NftablesOutputWriter(OutputWriter):
2604 def _write_footer(self):
2605 self.f.write("}\n")
2606
2607 - def _write_network(self, network):
2608 + def write(self, network):
2609 self.f.write(" %s,\n" % network)
2610
2611
2612 @@ -142,14 +119,9 @@ class XTGeoIPOutputWriter(OutputWriter):
2613 suffix = "iv"
2614 mode = "wb"
2615
2616 - def _write_network(self, network):
2617 - for address in (network.first_address, network.last_address):
2618 - # Convert this into a string of bits
2619 - bytes = socket.inet_pton(
2620 - network.family, address,
2621 - )
2622 -
2623 - self.f.write(bytes)
2624 + def write(self, network):
2625 + self.f.write(network._first_address)
2626 + self.f.write(network._last_address)
2627
2628
2629 formats = {
2630 @@ -185,8 +157,14 @@ class Exporter(object):
2631
2632 writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
2633
2634 + # Filter countries from special country codes
2635 + country_codes = [
2636 + country_code for country_code in countries if not country_code in FLAGS.values()
2637 + ]
2638 +
2639 # Get all networks that match the family
2640 - networks = self.db.search_networks(family=family)
2641 + networks = self.db.search_networks(family=family,
2642 + country_codes=country_codes, asns=asns, flatten=True)
2643
2644 # Walk through all networks
2645 for network in networks:
2646 @@ -203,10 +181,10 @@ class Exporter(object):
2647 pass
2648
2649 # Handle flags
2650 - for flag in flags:
2651 + for flag in FLAGS:
2652 if network.has_flag(flag):
2653 # Fetch the "fake" country code
2654 - country = flags[flag]
2655 + country = FLAGS[flag]
2656
2657 try:
2658 writers[country].write(network)
2659 diff --git a/src/python/importer.py b/src/python/importer.py
2660 index f19db4b..5f46bc3 100644
2661 --- a/src/python/importer.py
2662 +++ b/src/python/importer.py
2663 @@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
2664 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
2665
2666 # Latin America and Caribbean Network Information Centre
2667 - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2668 + "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2669
2670 # Réseaux IP Européens
2671 #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
2672 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2673 index 1467923..2dec89e 100644
2674 --- a/src/python/location-importer.in
2675 +++ b/src/python/location-importer.in
2676 @@ -152,6 +152,7 @@ class CLI(object):
2677 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
2678 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
2679 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
2680 + CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
2681
2682 -- autnums
2683 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
2684 @@ -165,6 +166,7 @@ class CLI(object):
2685 -- networks
2686 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
2687 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
2688 + CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
2689 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
2690
2691 -- overrides
2692 @@ -188,6 +190,8 @@ class CLI(object):
2693 );
2694 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
2695 ON network_overrides(network);
2696 + CREATE INDEX IF NOT EXISTS network_overrides_search
2697 + ON network_overrides USING GIST(network inet_ops);
2698 """)
2699
2700 return db
2701 @@ -234,32 +238,24 @@ class CLI(object):
2702
2703 # Select all known networks
2704 rows = self.db.query("""
2705 - -- Get a (sorted) list of all known networks
2706 - WITH known_networks AS (
2707 - SELECT network FROM announcements
2708 - UNION
2709 - SELECT network FROM networks
2710 - ORDER BY network
2711 - )
2712 -
2713 -- Return a list of those networks enriched with all
2714 -- other information that we store in the database
2715 SELECT
2716 - DISTINCT ON (known_networks.network)
2717 - known_networks.network AS network,
2718 - announcements.autnum AS autnum,
2719 + DISTINCT ON (network)
2720 + network,
2721 + autnum,
2722
2723 -- Country
2724 COALESCE(
2725 (
2726 SELECT country FROM network_overrides overrides
2727 - WHERE announcements.network <<= overrides.network
2728 + WHERE networks.network <<= overrides.network
2729 ORDER BY masklen(overrides.network) DESC
2730 LIMIT 1
2731 ),
2732 (
2733 SELECT country FROM autnum_overrides overrides
2734 - WHERE announcements.autnum = overrides.number
2735 + WHERE networks.autnum = overrides.number
2736 ),
2737 networks.country
2738 ) AS country,
2739 @@ -268,50 +264,67 @@ class CLI(object):
2740 COALESCE(
2741 (
2742 SELECT is_anonymous_proxy FROM network_overrides overrides
2743 - WHERE announcements.network <<= overrides.network
2744 + WHERE networks.network <<= overrides.network
2745 ORDER BY masklen(overrides.network) DESC
2746 LIMIT 1
2747 ),
2748 (
2749 SELECT is_anonymous_proxy FROM autnum_overrides overrides
2750 - WHERE announcements.autnum = overrides.number
2751 + WHERE networks.autnum = overrides.number
2752 ),
2753 FALSE
2754 ) AS is_anonymous_proxy,
2755 COALESCE(
2756 (
2757 SELECT is_satellite_provider FROM network_overrides overrides
2758 - WHERE announcements.network <<= overrides.network
2759 + WHERE networks.network <<= overrides.network
2760 ORDER BY masklen(overrides.network) DESC
2761 LIMIT 1
2762 ),
2763 (
2764 SELECT is_satellite_provider FROM autnum_overrides overrides
2765 - WHERE announcements.autnum = overrides.number
2766 + WHERE networks.autnum = overrides.number
2767 ),
2768 FALSE
2769 ) AS is_satellite_provider,
2770 COALESCE(
2771 (
2772 SELECT is_anycast FROM network_overrides overrides
2773 - WHERE announcements.network <<= overrides.network
2774 + WHERE networks.network <<= overrides.network
2775 ORDER BY masklen(overrides.network) DESC
2776 LIMIT 1
2777 ),
2778 (
2779 SELECT is_anycast FROM autnum_overrides overrides
2780 - WHERE announcements.autnum = overrides.number
2781 + WHERE networks.autnum = overrides.number
2782 ),
2783 FALSE
2784 - ) AS is_anycast,
2785 -
2786 - -- Must be part of returned values for ORDER BY clause
2787 - masklen(announcements.network) AS sort_a,
2788 - masklen(networks.network) AS sort_b
2789 - FROM known_networks
2790 - LEFT JOIN announcements ON known_networks.network <<= announcements.network
2791 - LEFT JOIN networks ON known_networks.network <<= networks.network
2792 - ORDER BY known_networks.network, sort_a DESC, sort_b DESC
2793 + ) AS is_anycast
2794 + FROM (
2795 + SELECT
2796 + known_networks.network AS network,
2797 + announcements.autnum AS autnum,
2798 + networks.country AS country,
2799 +
2800 + -- Must be part of returned values for ORDER BY clause
2801 + masklen(announcements.network) AS sort_a,
2802 + masklen(networks.network) AS sort_b
2803 + FROM (
2804 + SELECT network FROM announcements
2805 + UNION ALL
2806 + SELECT network FROM networks
2807 + UNION ALL
2808 + SELECT network FROM network_overrides
2809 + ) known_networks
2810 + LEFT JOIN
2811 + announcements ON known_networks.network <<= announcements.network
2812 + LEFT JOIN
2813 + networks ON known_networks.network <<= networks.network
2814 + ORDER BY
2815 + known_networks.network,
2816 + sort_a DESC,
2817 + sort_b DESC
2818 + ) networks
2819 """)
2820
2821 for row in rows:
2822 @@ -363,6 +376,16 @@ class CLI(object):
2823 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
2824 ON COMMIT DROP;
2825 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
2826 +
2827 + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
2828 + ON COMMIT DROP;
2829 + CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
2830 + CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
2831 + """)
2832 +
2833 + # Remove all previously imported content
2834 + self.db.execute("""
2835 + TRUNCATE TABLE networks;
2836 """)
2837
2838 for source in location.importer.WHOIS_SOURCES:
2839 @@ -370,31 +393,72 @@ class CLI(object):
2840 for block in f:
2841 self._parse_block(block)
2842
2843 + # Process all parsed networks from every RIR we happen to have access to,
2844 + # insert the largest network chunks into the networks table immediately...
2845 + families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
2846 +
2847 + for family in (row.family for row in families):
2848 + smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
2849 +
2850 + self.db.execute("INSERT INTO networks(network, country) \
2851 + SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
2852 +
2853 + # ... determine any other prefixes for this network family, ...
2854 + prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
2855 + WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
2856 +
2857 + # ... and insert networks with this prefix in case they provide additional
2858 + # information (i. e. subnet of a larger chunk with a different country)
2859 + for prefix in (row.prefix for row in prefixes):
2860 + self.db.execute("""
2861 + WITH candidates AS (
2862 + SELECT
2863 + _rirdata.network,
2864 + _rirdata.country
2865 + FROM
2866 + _rirdata
2867 + WHERE
2868 + family(_rirdata.network) = %s
2869 + AND
2870 + masklen(_rirdata.network) = %s
2871 + ),
2872 + filtered AS (
2873 + SELECT
2874 + DISTINCT ON (c.network)
2875 + c.network,
2876 + c.country,
2877 + masklen(networks.network),
2878 + networks.country AS parent_country
2879 + FROM
2880 + candidates c
2881 + LEFT JOIN
2882 + networks
2883 + ON
2884 + c.network << networks.network
2885 + ORDER BY
2886 + c.network,
2887 + masklen(networks.network) DESC NULLS LAST
2888 + )
2889 + INSERT INTO
2890 + networks(network, country)
2891 + SELECT
2892 + network,
2893 + country
2894 + FROM
2895 + filtered
2896 + WHERE
2897 + parent_country IS NULL
2898 + OR
2899 + country <> parent_country
2900 + ON CONFLICT DO NOTHING""",
2901 + family, prefix,
2902 + )
2903 +
2904 self.db.execute("""
2905 INSERT INTO autnums(number, name)
2906 SELECT _autnums.number, _organizations.name FROM _autnums
2907 JOIN _organizations ON _autnums.organization = _organizations.handle
2908 - ON CONFLICT (number) DO UPDATE SET name = excluded.name
2909 - """)
2910 -
2911 - self.db.execute("""
2912 - --- Purge any redundant entries
2913 - CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
2914 - AS
2915 - SELECT network FROM networks candidates
2916 - WHERE EXISTS (
2917 - SELECT FROM networks
2918 - WHERE
2919 - networks.network << candidates.network
2920 - AND
2921 - networks.country = candidates.country
2922 - );
2923 -
2924 - CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
2925 -
2926 - DELETE FROM networks WHERE EXISTS (
2927 - SELECT FROM _garbage WHERE networks.network = _garbage.network
2928 - );
2929 + ON CONFLICT (number) DO UPDATE SET name = excluded.name;
2930 """)
2931
2932 # Download all extended sources
2933 @@ -405,6 +469,69 @@ class CLI(object):
2934 for line in f:
2935 self._parse_line(line)
2936
2937 + def _check_parsed_network(self, network):
2938 + """
2939 + Assistive function to detect and subsequently sort out parsed
2940 + networks from RIR data (both Whois and so-called "extended sources"),
2941 + which are or have...
2942 +
2943 + (a) not globally routable (RFC 1918 space, et al.)
2944 + (b) covering a too large chunk of the IP address space (prefix length
2945 + is < 7 for IPv4 networks, and < 10 for IPv6)
2946 + (c) "0.0.0.0" or "::" as a network address
2947 + (d) are too small for being publicly announced (we have decided not to
2948 + process them at the moment, as they significantly enlarge our
2949 + database without providing very helpful additional information)
2950 +
2951 + This unfortunately is necessary due to brain-dead clutter across
2952 + various RIR databases, causing mismatches and eventually disruptions.
2953 +
2954 + We will return False in case a network is not suitable for adding
2955 + it to our database, and True otherwise.
2956 + """
2957 +
2958 + if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
2959 + return False
2960 +
2961 + if not network.is_global:
2962 + log.warning("Skipping non-globally routable network: %s" % network)
2963 + return False
2964 +
2965 + if network.version == 4:
2966 + if network.prefixlen < 7:
2967 + log.warning("Skipping too big IP chunk: %s" % network)
2968 + return False
2969 +
2970 + if network.prefixlen > 24:
2971 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2972 + return False
2973 +
2974 + if str(network.network_address) == "0.0.0.0":
2975 + log.warning("Skipping network based on 0.0.0.0: %s" % network)
2976 + return False
2977 +
2978 + elif network.version == 6:
2979 + if network.prefixlen < 10:
2980 + log.warning("Skipping too big IP chunk: %s" % network)
2981 + return False
2982 +
2983 + if network.prefixlen > 48:
2984 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2985 + return False
2986 +
2987 + if str(network.network_address) == "::":
2988 + log.warning("Skipping network based on '::': %s" % network)
2989 + return False
2990 +
2991 + else:
2992 + # This should not happen...
2993 + log.warning("Skipping network of unknown family, this should not happen: %s" % network)
2994 + return False
2995 +
2996 + # In case we have made it here, the network is considered to
2997 + # be suitable for libloc consumption...
2998 + return True
2999 +
3000 def _parse_block(self, block):
3001 # Get first line to find out what type of block this is
3002 line = block[0]
3003 @@ -433,7 +560,7 @@ class CLI(object):
3004 autnum["asn"] = m.group(2)
3005
3006 elif key == "org":
3007 - autnum[key] = val
3008 + autnum[key] = val.upper()
3009
3010 # Skip empty objects
3011 if not autnum:
3012 @@ -447,15 +574,22 @@ class CLI(object):
3013 )
3014
3015 def _parse_inetnum_block(self, block):
3016 - logging.debug("Parsing inetnum block:")
3017 + log.debug("Parsing inetnum block:")
3018
3019 inetnum = {}
3020 for line in block:
3021 - logging.debug(line)
3022 + log.debug(line)
3023
3024 # Split line
3025 key, val = split_line(line)
3026
3027 + # Filter any inetnum records which are only referring to IP space
3028 + # not managed by that specific RIR...
3029 + if key == "netname":
3030 + if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
3031 + log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
3032 + return
3033 +
3034 if key == "inetnum":
3035 start_address, delim, end_address = val.partition("-")
3036
3037 @@ -467,7 +601,7 @@ class CLI(object):
3038 start_address = ipaddress.ip_address(start_address)
3039 end_address = ipaddress.ip_address(end_address)
3040 except ValueError:
3041 - logging.warning("Could not parse line: %s" % line)
3042 + log.warning("Could not parse line: %s" % line)
3043 return
3044
3045 # Set prefix to default
3046 @@ -484,23 +618,24 @@ class CLI(object):
3047 inetnum[key] = val
3048
3049 elif key == "country":
3050 - if val == "UNITED STATES":
3051 - val = "US"
3052 -
3053 inetnum[key] = val.upper()
3054
3055 # Skip empty objects
3056 - if not inetnum:
3057 + if not inetnum or not "country" in inetnum:
3058 + return
3059 +
3060 + # Skip objects with bogus country code 'ZZ'
3061 + if inetnum.get("country") == "ZZ":
3062 + log.warning("Skipping network with bogus country 'ZZ': %s" % \
3063 + (inetnum.get("inet6num") or inetnum.get("inetnum")))
3064 return
3065
3066 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
3067
3068 - # Bail out in case we have processed a non-public IP network
3069 - if network.is_private:
3070 - logging.warning("Skipping non-globally routable network: %s" % network)
3071 + if not self._check_parsed_network(network):
3072 return
3073
3074 - self.db.execute("INSERT INTO networks(network, country) \
3075 + self.db.execute("INSERT INTO _rirdata(network, country) \
3076 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
3077 "%s" % network, inetnum.get("country"),
3078 )
3079 @@ -511,7 +646,9 @@ class CLI(object):
3080 # Split line
3081 key, val = split_line(line)
3082
3083 - if key in ("organisation", "org-name"):
3084 + if key == "organisation":
3085 + org[key] = val.upper()
3086 + elif key == "org-name":
3087 org[key] = val
3088
3089 # Skip empty objects
3090 @@ -581,6 +718,9 @@ class CLI(object):
3091 log.warning("Invalid IP address: %s" % address)
3092 return
3093
3094 + if not self._check_parsed_network(network):
3095 + return
3096 +
3097 self.db.execute("INSERT INTO networks(network, country) \
3098 VALUES(%s, %s) ON CONFLICT (network) DO \
3099 UPDATE SET country = excluded.country",
3100 diff --git a/src/python/location.in b/src/python/location.in
3101 index 44ad726..b30beae 100644
3102 --- a/src/python/location.in
3103 +++ b/src/python/location.in
3104 @@ -253,6 +253,7 @@ class CLI(object):
3105 network = db.lookup(address)
3106 except ValueError:
3107 print(_("Invalid IP address: %s") % address, file=sys.stderr)
3108 + return 2
3109
3110 args = {
3111 "address" : address,
3112 @@ -398,10 +399,7 @@ class CLI(object):
3113
3114 def handle_update(self, db, ns):
3115 if ns.cron and db:
3116 - now = datetime.datetime.utcnow()
3117 -
3118 - # Parse the database timestamp
3119 - t = datetime.datetime.utcfromtimestamp(db.created_at)
3120 + now = time.time()
3121
3122 if ns.cron == "daily":
3123 delta = datetime.timedelta(days=1)
3124 @@ -410,22 +408,20 @@ class CLI(object):
3125 elif ns.cron == "monthly":
3126 delta = datetime.timedelta(days=30)
3127
3128 + delta = delta.total_seconds()
3129 +
3130 # Check if the database has recently been updated
3131 - if t >= (now - delta):
3132 + if db.created_at >= (now - delta):
3133 log.info(
3134 - _("The database has been updated recently (%s)") % \
3135 - format_timedelta(now - t),
3136 + _("The database has been updated recently"),
3137 )
3138 return 3
3139
3140 # Fetch the timestamp we need from DNS
3141 t = location.discover_latest_version()
3142
3143 - # Parse timestamp into datetime format
3144 - timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
3145 -
3146 # Check the version of the local database
3147 - if db and timestamp and db.created_at >= timestamp.timestamp():
3148 + if db and t and db.created_at >= t:
3149 log.info("Already on the latest version")
3150 return
3151
3152 @@ -437,7 +433,7 @@ class CLI(object):
3153
3154 # Try downloading a new database
3155 try:
3156 - t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
3157 + t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
3158
3159 # If no file could be downloaded, log a message
3160 except FileNotFoundError as e:
3161 @@ -453,13 +449,7 @@ class CLI(object):
3162
3163 return 0
3164
3165 - def handle_verify(self, ns):
3166 - try:
3167 - db = location.Database(ns.database)
3168 - except FileNotFoundError as e:
3169 - log.error("%s: %s" % (ns.database, e))
3170 - return 127
3171 -
3172 + def handle_verify(self, db, ns):
3173 # Verify the database
3174 with open(ns.public_key, "r") as f:
3175 if not db.verify(f):
3176 diff --git a/src/python/network.c b/src/python/network.c
3177 index 5496d1e..5b1369d 100644
3178 --- a/src/python/network.c
3179 +++ b/src/python/network.c
3180 @@ -17,13 +17,33 @@
3181 #include <Python.h>
3182
3183 #include <errno.h>
3184 +#include <limits.h>
3185
3186 #include <loc/libloc.h>
3187 #include <loc/network.h>
3188 +#include <loc/network-list.h>
3189
3190 #include "locationmodule.h"
3191 #include "network.h"
3192
3193 +static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
3194 + PyObject* list = PyList_New(0);
3195 + if (!networks)
3196 + return list;
3197 +
3198 + while (!loc_network_list_empty(networks)) {
3199 + struct loc_network* network = loc_network_list_pop(networks);
3200 +
3201 + PyObject* n = new_network(&NetworkType, network);
3202 + PyList_Append(list, n);
3203 +
3204 + loc_network_unref(network);
3205 + Py_DECREF(n);
3206 + }
3207 +
3208 + return list;
3209 +}
3210 +
3211 PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
3212 NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
3213 if (self) {
3214 @@ -114,10 +134,18 @@ static int Network_set_asn(NetworkObject* self, PyObject* value) {
3215 long int asn = PyLong_AsLong(value);
3216
3217 // Check if the ASN is within the valid range
3218 - if (asn <= 0 || asn > UINT32_MAX) {
3219 + if (asn <= 0) {
3220 + PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn);
3221 + return -1;
3222 + }
3223 +
3224 +#if (__WORDSIZE > 32)
3225 + // Check whether the input was longer than 32 bit
3226 + if (asn > UINT32_MAX) {
3227 PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn);
3228 return -1;
3229 }
3230 +#endif
3231
3232 int r = loc_network_set_asn(self->network, asn);
3233 if (r)
3234 @@ -154,13 +182,28 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
3235 Py_RETURN_NONE;
3236 }
3237
3238 +static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
3239 + NetworkObject* other = NULL;
3240 +
3241 + if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3242 + return NULL;
3243 +
3244 + struct loc_network_list* list = loc_network_exclude(self->network, other->network);
3245 +
3246 + // Convert to Python objects
3247 + PyObject* obj = PyList_FromNetworkList(list);
3248 + loc_network_list_unref(list);
3249 +
3250 + return obj;
3251 +}
3252 +
3253 static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
3254 NetworkObject* other = NULL;
3255
3256 if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3257 return NULL;
3258
3259 - if (loc_network_is_subnet_of(self->network, other->network))
3260 + if (loc_network_is_subnet(other->network, self->network))
3261 Py_RETURN_TRUE;
3262
3263 Py_RETURN_FALSE;
3264 @@ -181,6 +224,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) {
3265 return obj;
3266 }
3267
3268 +static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) {
3269 + struct in_addr address4;
3270 +
3271 + // Convert IPv4 addresses to struct in_addr
3272 + if (IN6_IS_ADDR_V4MAPPED(address6)) {
3273 + address4.s_addr = address6->s6_addr32[3];
3274 +
3275 + return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4));
3276 + }
3277 +
3278 + // Return IPv6 addresses as they are
3279 + return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6));
3280 +}
3281 +
3282 +static PyObject* Network_get__first_address(NetworkObject* self) {
3283 + const struct in6_addr* address = loc_network_get_first_address(self->network);
3284 +
3285 + return PyBytes_FromAddress(address);
3286 +}
3287 +
3288 static PyObject* Network_get_last_address(NetworkObject* self) {
3289 char* address = loc_network_format_last_address(self->network);
3290
3291 @@ -190,7 +253,19 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
3292 return obj;
3293 }
3294
3295 +static PyObject* Network_get__last_address(NetworkObject* self) {
3296 + const struct in6_addr* address = loc_network_get_last_address(self->network);
3297 +
3298 + return PyBytes_FromAddress(address);
3299 +}
3300 +
3301 static struct PyMethodDef Network_methods[] = {
3302 + {
3303 + "exclude",
3304 + (PyCFunction)Network_exclude,
3305 + METH_VARARGS,
3306 + NULL,
3307 + },
3308 {
3309 "has_flag",
3310 (PyCFunction)Network_has_flag,
3311 @@ -241,6 +316,13 @@ static struct PyGetSetDef Network_getsetters[] = {
3312 NULL,
3313 NULL,
3314 },
3315 + {
3316 + "_first_address",
3317 + (getter)Network_get__first_address,
3318 + NULL,
3319 + NULL,
3320 + NULL,
3321 + },
3322 {
3323 "last_address",
3324 (getter)Network_get_last_address,
3325 @@ -248,6 +330,13 @@ static struct PyGetSetDef Network_getsetters[] = {
3326 NULL,
3327 NULL,
3328 },
3329 + {
3330 + "_last_address",
3331 + (getter)Network_get__last_address,
3332 + NULL,
3333 + NULL,
3334 + NULL,
3335 + },
3336 { NULL },
3337 };
3338
3339 diff --git a/src/test-as.c b/src/test-as.c
3340 index 839a04c..2d61675 100644
3341 --- a/src/test-as.c
3342 +++ b/src/test-as.c
3343 @@ -95,7 +95,7 @@ int main(int argc, char** argv) {
3344 // Enumerator
3345
3346 struct loc_database_enumerator* enumerator;
3347 - err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
3348 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
3349 if (err) {
3350 fprintf(stderr, "Could not create a database enumerator\n");
3351 exit(EXIT_FAILURE);
3352 diff --git a/src/test-database.c b/src/test-database.c
3353 index b4a75c4..da4b11c 100644
3354 --- a/src/test-database.c
3355 +++ b/src/test-database.c
3356 @@ -38,6 +38,14 @@ const char* DESCRIPTION =
3357 "Maecenas ut venenatis nunc.";
3358 const char* LICENSE = "CC";
3359
3360 +const char* networks[] = {
3361 + "2001:db8::/32",
3362 + "2001:db8:1000::/48",
3363 + "2001:db8:2000::/48",
3364 + "2001:db8:2020::/48",
3365 + NULL,
3366 +};
3367 +
3368 static int attempt_to_open(struct loc_ctx* ctx, char* path) {
3369 FILE* f = fopen(path, "r");
3370 if (!f)
3371 @@ -139,6 +147,24 @@ int main(int argc, char** argv) {
3372 exit(EXIT_FAILURE);
3373 }
3374
3375 + struct loc_network* network = NULL;
3376 +
3377 + // Add some networks
3378 + const char** n = networks;
3379 + while (*n) {
3380 + err = loc_writer_add_network(writer, &network, *n);
3381 + if (err) {
3382 + fprintf(stderr, "Could not add network %s\n", *n);
3383 + exit(EXIT_FAILURE);
3384 + }
3385 +
3386 + // Set a country
3387 + loc_network_set_country_code(network, "XX");
3388 +
3389 + // Next one
3390 + n++;
3391 + }
3392 +
3393 FILE* f = tmpfile();
3394 if (!f) {
3395 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
3396 @@ -170,6 +196,33 @@ int main(int argc, char** argv) {
3397 exit(EXIT_FAILURE);
3398 }
3399
3400 + // Enumerator
3401 + struct loc_database_enumerator* enumerator;
3402 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
3403 + if (err) {
3404 + fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
3405 + exit(EXIT_FAILURE);
3406 + }
3407 +
3408 + // Walk through all networks
3409 + while (1) {
3410 + err = loc_database_enumerator_next_network(enumerator, &network);
3411 + if (err) {
3412 + fprintf(stderr, "Error fetching the next network: %d\n", err);
3413 + exit(EXIT_FAILURE);
3414 + }
3415 +
3416 + if (!network)
3417 + break;
3418 +
3419 + char* s = loc_network_str(network);
3420 + printf("Got network: %s\n", s);
3421 + free(s);
3422 + }
3423 +
3424 + // Free the enumerator
3425 + loc_database_enumerator_unref(enumerator);
3426 +
3427 // Close the database
3428 loc_database_unref(db);
3429 loc_unref(ctx);
3430 diff --git a/src/test-network-list.c b/src/test-network-list.c
3431 new file mode 100644
3432 index 0000000..6f32ff7
3433 --- /dev/null
3434 +++ b/src/test-network-list.c
3435 @@ -0,0 +1,183 @@
3436 +/*
3437 + libloc - A library to determine the location of someone on the Internet
3438 +
3439 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
3440 +
3441 + This program is free software; you can redistribute it and/or modify
3442 + it under the terms of the GNU General Public License as published by
3443 + the Free Software Foundation; either version 2 of the License, or
3444 + (at your option) any later version.
3445 +
3446 + This program is distributed in the hope that it will be useful,
3447 + but WITHOUT ANY WARRANTY; without even the implied warranty of
3448 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3449 + GNU General Public License for more details.
3450 +*/
3451 +
3452 +#include <errno.h>
3453 +#include <stdio.h>
3454 +#include <stddef.h>
3455 +#include <stdlib.h>
3456 +#include <string.h>
3457 +#include <syslog.h>
3458 +
3459 +#include <loc/libloc.h>
3460 +#include <loc/network.h>
3461 +#include <loc/network-list.h>
3462 +
3463 +int main(int argc, char** argv) {
3464 + int err;
3465 +
3466 + struct loc_ctx* ctx;
3467 + err = loc_new(&ctx);
3468 + if (err < 0)
3469 + exit(EXIT_FAILURE);
3470 +
3471 + // Enable debug logging
3472 + loc_set_log_priority(ctx, LOG_DEBUG);
3473 +
3474 + // Create a network
3475 + struct loc_network* network1;
3476 + err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32");
3477 + if (err) {
3478 + fprintf(stderr, "Could not create the network1\n");
3479 + exit(EXIT_FAILURE);
3480 + }
3481 +
3482 + struct loc_network* subnet1;
3483 + err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48");
3484 + if (err) {
3485 + fprintf(stderr, "Could not create the subnet1\n");
3486 + exit(EXIT_FAILURE);
3487 + }
3488 +
3489 + struct loc_network* subnet2;
3490 + err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48");
3491 + if (err) {
3492 + fprintf(stderr, "Could not create the subnet2\n");
3493 + exit(EXIT_FAILURE);
3494 + }
3495 +
3496 + struct loc_network* subnet3;
3497 + err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48");
3498 + if (err) {
3499 + fprintf(stderr, "Could not create the subnet3\n");
3500 + exit(EXIT_FAILURE);
3501 + }
3502 +
3503 + struct loc_network* subnet4;
3504 + err = loc_network_new_from_string(ctx, &subnet4, "2001:db8:d::/48");
3505 + if (err) {
3506 + fprintf(stderr, "Could not create the subnet4\n");
3507 + exit(EXIT_FAILURE);
3508 + }
3509 +
3510 + struct loc_network* subnet5;
3511 + err = loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48");
3512 + if (err) {
3513 + fprintf(stderr, "Could not create the subnet5\n");
3514 + exit(EXIT_FAILURE);
3515 + }
3516 +
3517 + struct loc_network* subnet6;
3518 + err = loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48");
3519 + if (err) {
3520 + fprintf(stderr, "Could not create the subnet6\n");
3521 + exit(EXIT_FAILURE);
3522 + }
3523 +
3524 + // Make a list with both subnets
3525 + struct loc_network_list* subnets;
3526 + err = loc_network_list_new(ctx, &subnets);
3527 + if (err) {
3528 + fprintf(stderr, "Could not create subnets list\n");
3529 + exit(EXIT_FAILURE);
3530 + }
3531 +
3532 + size_t size = loc_network_list_size(subnets);
3533 + if (size > 0) {
3534 + fprintf(stderr, "The list is not empty: %zu\n", size);
3535 + exit(EXIT_FAILURE);
3536 + }
3537 +
3538 + err = loc_network_list_push(subnets, subnet1);
3539 + if (err) {
3540 + fprintf(stderr, "Could not add subnet1 to subnets list\n");
3541 + exit(EXIT_FAILURE);
3542 + }
3543 +
3544 + if (loc_network_list_empty(subnets)) {
3545 + fprintf(stderr, "The subnets list reports that it is empty\n");
3546 + exit(EXIT_FAILURE);
3547 + }
3548 +
3549 + err = loc_network_list_push(subnets, subnet2);
3550 + if (err) {
3551 + fprintf(stderr, "Could not add subnet2 to subnets list\n");
3552 + exit(EXIT_FAILURE);
3553 + }
3554 +
3555 + // Add the fourth one next
3556 + err = loc_network_list_push(subnets, subnet4);
3557 + if (err) {
3558 + fprintf(stderr, "Could not add subnet4 to subnets list\n");
3559 + exit(EXIT_FAILURE);
3560 + }
3561 +
3562 + // Add the third one
3563 + err = loc_network_list_push(subnets, subnet3);
3564 + if (err) {
3565 + fprintf(stderr, "Could not add subnet3 to subnets list\n");
3566 + exit(EXIT_FAILURE);
3567 + }
3568 +
3569 + // Add more subnets
3570 + err = loc_network_list_push(subnets, subnet5);
3571 + if (err) {
3572 + fprintf(stderr, "Could not add subnet5 to subnets list\n");
3573 + exit(EXIT_FAILURE);
3574 + }
3575 +
3576 + err = loc_network_list_push(subnets, subnet6);
3577 + if (err) {
3578 + fprintf(stderr, "Could not add subnet6 to subnets list\n");
3579 + exit(EXIT_FAILURE);
3580 + }
3581 +
3582 + loc_network_list_dump(subnets);
3583 +
3584 + size = loc_network_list_size(subnets);
3585 + if (size != 6) {
3586 + fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
3587 + exit(EXIT_FAILURE);
3588 + }
3589 +
3590 + // Exclude subnet1 from network1
3591 + struct loc_network_list* excluded = loc_network_exclude(network1, subnet1);
3592 + if (!excluded) {
3593 + fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n");
3594 + exit(EXIT_FAILURE);
3595 + }
3596 +
3597 + loc_network_list_dump(excluded);
3598 +
3599 + // Exclude all subnets from network1
3600 + excluded = loc_network_exclude_list(network1, subnets);
3601 + if (!excluded) {
3602 + fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n");
3603 + exit(EXIT_FAILURE);
3604 + }
3605 +
3606 + loc_network_list_dump(excluded);
3607 +
3608 + if (excluded)
3609 + loc_network_list_unref(excluded);
3610 +
3611 + loc_network_list_unref(subnets);
3612 + loc_network_unref(network1);
3613 + loc_network_unref(subnet1);
3614 + loc_network_unref(subnet2);
3615 + loc_unref(ctx);
3616 +
3617 + return EXIT_SUCCESS;
3618 +}
3619 diff --git a/src/test-network.c b/src/test-network.c
3620 index d38f13d..dde13f1 100644
3621 --- a/src/test-network.c
3622 +++ b/src/test-network.c
3623 @@ -14,6 +14,7 @@
3624 GNU General Public License for more details.
3625 */
3626
3627 +#include <arpa/inet.h>
3628 #include <errno.h>
3629 #include <stdio.h>
3630 #include <stddef.h>
3631 @@ -37,12 +38,21 @@ int main(int argc, char** argv) {
3632 // Enable debug logging
3633 loc_set_log_priority(ctx, LOG_DEBUG);
3634
3635 +#if 0
3636 struct loc_network_tree* tree;
3637 err = loc_network_tree_new(ctx, &tree);
3638 if (err) {
3639 fprintf(stderr, "Could not create the network tree\n");
3640 exit(EXIT_FAILURE);
3641 }
3642 +#endif
3643 +
3644 + struct in6_addr address;
3645 + err = inet_pton(AF_INET6, "2001:db8::1", &address);
3646 + if (err != 1) {
3647 + fprintf(stderr, "Could not parse IP address\n");
3648 + exit(EXIT_FAILURE);
3649 + }
3650
3651 // Create a network
3652 struct loc_network* network1;
3653 @@ -58,12 +68,14 @@ int main(int argc, char** argv) {
3654 exit(EXIT_FAILURE);
3655 }
3656
3657 +#if 0
3658 // Adding network to the tree
3659 err = loc_network_tree_add_network(tree, network1);
3660 if (err) {
3661 fprintf(stderr, "Could not add network to the tree\n");
3662 exit(EXIT_FAILURE);
3663 }
3664 +#endif
3665
3666 // Check if the first and last addresses are correct
3667 char* string = loc_network_format_first_address(network1);
3668 @@ -88,6 +100,12 @@ int main(int argc, char** argv) {
3669 exit(EXIT_FAILURE);
3670 }
3671
3672 + err = loc_network_match_address(network1, &address);
3673 + if (!err) {
3674 + fprintf(stderr, "Network1 does not match address\n");
3675 + exit(EXIT_FAILURE);
3676 + }
3677 +
3678 struct loc_network* network2;
3679 err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48");
3680 if (err) {
3681 @@ -101,6 +119,7 @@ int main(int argc, char** argv) {
3682 exit(EXIT_FAILURE);
3683 }
3684
3685 +#if 0
3686 // Adding network to the tree
3687 err = loc_network_tree_add_network(tree, network2);
3688 if (err) {
3689 @@ -117,20 +136,84 @@ int main(int argc, char** argv) {
3690
3691 size_t nodes = loc_network_tree_count_nodes(tree);
3692 printf("The tree has %zu nodes\n", nodes);
3693 +#endif
3694 +
3695 + // Check equals function
3696 + err = loc_network_cmp(network1, network1);
3697 + if (err) {
3698 + fprintf(stderr, "Network is not equal with itself\n");
3699 + exit(EXIT_FAILURE);
3700 + }
3701 +
3702 + err = loc_network_cmp(network1, network2);
3703 + if (!err) {
3704 + fprintf(stderr, "Networks equal unexpectedly\n");
3705 + exit(EXIT_FAILURE);
3706 + }
3707
3708 // Check subnet function
3709 - err = loc_network_is_subnet_of(network1, network2);
3710 - if (err != 0) {
3711 + err = loc_network_is_subnet(network1, network2);
3712 + if (!err) {
3713 fprintf(stderr, "Subnet check 1 failed: %d\n", err);
3714 exit(EXIT_FAILURE);
3715 }
3716
3717 - err = loc_network_is_subnet_of(network2, network1);
3718 - if (err != 1) {
3719 + err = loc_network_is_subnet(network2, network1);
3720 + if (err) {
3721 fprintf(stderr, "Subnet check 2 failed: %d\n", err);
3722 exit(EXIT_FAILURE);
3723 }
3724
3725 + // Make subnets
3726 + struct loc_network* subnet1 = NULL;
3727 + struct loc_network* subnet2 = NULL;
3728 +
3729 + err = loc_network_subnets(network1, &subnet1, &subnet2);
3730 + if (err || !subnet1 || !subnet2) {
3731 + fprintf(stderr, "Could not find subnets of network: %d\n", err);
3732 + exit(EXIT_FAILURE);
3733 + }
3734 +
3735 + char* s = loc_network_str(subnet1);
3736 + printf("Received subnet1 = %s\n", s);
3737 + free(s);
3738 +
3739 + s = loc_network_str(subnet2);
3740 + printf("Received subnet2 = %s\n", s);
3741 + free(s);
3742 +
3743 + if (!loc_network_is_subnet(network1, subnet1)) {
3744 + fprintf(stderr, "Subnet1 is not a subnet\n");
3745 + exit(EXIT_FAILURE);
3746 + }
3747 +
3748 + if (!loc_network_is_subnet(network1, subnet2)) {
3749 + fprintf(stderr, "Subnet2 is not a subnet\n");
3750 + exit(EXIT_FAILURE);
3751 + }
3752 +
3753 + if (!loc_network_overlaps(network1, subnet1)) {
3754 + fprintf(stderr, "Network1 does not seem to contain subnet1\n");
3755 + exit(EXIT_FAILURE);
3756 + }
3757 +
3758 + if (!loc_network_overlaps(network1, subnet2)) {
3759 + fprintf(stderr, "Network1 does not seem to contain subnet2\n");
3760 + exit(EXIT_FAILURE);
3761 + }
3762 +
3763 + loc_network_unref(subnet1);
3764 + loc_network_unref(subnet2);
3765 +
3766 + struct loc_network_list* excluded = loc_network_exclude(network1, network2);
3767 + if (!excluded) {
3768 + fprintf(stderr, "Could not create excluded list\n");
3769 + exit(EXIT_FAILURE);
3770 + }
3771 +
3772 + loc_network_list_dump(excluded);
3773 + loc_network_list_unref(excluded);
3774 +
3775 // Create a database
3776 struct loc_writer* writer;
3777 err = loc_writer_new(ctx, &writer, NULL, NULL);
3778 @@ -160,6 +243,28 @@ int main(int argc, char** argv) {
3779 // Set ASN
3780 loc_network_set_asn(network4, 1024);
3781
3782 + // Try adding an invalid network
3783 + struct loc_network* network;
3784 + err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
3785 + if (err != -EINVAL) {
3786 + fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
3787 + exit(EXIT_FAILURE);
3788 + }
3789 +
3790 + // Try adding a single address
3791 + err = loc_writer_add_network(writer, &network, "2001:db8::");
3792 + if (err) {
3793 + fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
3794 + exit(EXIT_FAILURE);
3795 + }
3796 +
3797 + // Try adding localhost
3798 + err = loc_writer_add_network(writer, &network, "::1/128");
3799 + if (err != -EINVAL) {
3800 + fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
3801 + exit(EXIT_FAILURE);
3802 + }
3803 +
3804 FILE* f = tmpfile();
3805 if (!f) {
3806 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
3807 @@ -177,7 +282,10 @@ int main(int argc, char** argv) {
3808 loc_network_unref(network2);
3809 loc_network_unref(network3);
3810 loc_network_unref(network4);
3811 +
3812 +#if 0
3813 loc_network_tree_unref(tree);
3814 +#endif
3815
3816 // And open it again from disk
3817 struct loc_database* db;
3818 diff --git a/src/writer.c b/src/writer.c
3819 index 5939cff..c61a6df 100644
3820 --- a/src/writer.c
3821 +++ b/src/writer.c
3822 @@ -147,8 +147,19 @@ static void loc_writer_free(struct loc_writer* writer) {
3823 EVP_PKEY_free(writer->private_key2);
3824
3825 // Unref all AS
3826 - for (unsigned int i = 0; i < writer->as_count; i++) {
3827 - loc_as_unref(writer->as[i]);
3828 + if (writer->as) {
3829 + for (unsigned int i = 0; i < writer->as_count; i++) {
3830 + loc_as_unref(writer->as[i]);
3831 + }
3832 + free(writer->as);
3833 + }
3834 +
3835 + // Unref all countries
3836 + if (writer->countries) {
3837 + for (unsigned int i = 0; i < writer->countries_count; i++) {
3838 + loc_country_unref(writer->countries[i]);
3839 + }
3840 + free(writer->countries);
3841 }
3842
3843 // Release network tree
3844 @@ -601,7 +612,7 @@ static int loc_writer_create_signature(struct loc_writer* writer,
3845 goto END;
3846 }
3847
3848 - DEBUG(writer->ctx, "Successfully generated signature of %lu bytes\n", *length);
3849 + DEBUG(writer->ctx, "Successfully generated signature of %zu bytes\n", *length);
3850 r = 0;
3851
3852 // Dump signature
Peter's tree of IPFire 2.x