]> git.ipfire.org Git - people/pmueller/ipfire-2.x.git/blob - src/patches/libloc-0.9.4-upstream.patch
projects / people / pmueller / ipfire-2.x.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
libloc: Import latest changes from upstream
[people/pmueller/ipfire-2.x.git] / src / patches / libloc-0.9.4-upstream.patch
1 diff --git a/Makefile.am b/Makefile.am
2 index a0431a6..ebd7e17 100644
3 --- a/Makefile.am
4 +++ b/Makefile.am
5 @@ -91,11 +91,14 @@ EXTRA_DIST += \
6 pkginclude_HEADERS = \
7 src/loc/libloc.h \
8 src/loc/as.h \
9 + src/loc/as-list.h \
10 src/loc/compat.h \
11 src/loc/country.h \
12 + src/loc/country-list.h \
13 src/loc/database.h \
14 src/loc/format.h \
15 src/loc/network.h \
16 + src/loc/network-list.h \
17 src/loc/private.h \
18 src/loc/stringpool.h \
19 src/loc/resolv.h \
20 @@ -107,9 +110,12 @@ lib_LTLIBRARIES = \
21 src_libloc_la_SOURCES = \
22 src/libloc.c \
23 src/as.c \
24 + src/as-list.c \
25 src/country.c \
26 + src/country-list.c \
27 src/database.c \
28 src/network.c \
29 + src/network-list.c \
30 src/resolv.c \
31 src/stringpool.c \
32 src/writer.c
33 @@ -312,6 +318,7 @@ check_PROGRAMS = \
34 src/test-database \
35 src/test-as \
36 src/test-network \
37 + src/test-network-list \
38 src/test-country \
39 src/test-signature
40
41 @@ -351,6 +358,15 @@ src_test_network_CFLAGS = \
42 src_test_network_LDADD = \
43 src/libloc.la
44
45 +src_test_network_list_SOURCES = \
46 + src/test-network-list.c
47 +
48 +src_test_network_list_CFLAGS = \
49 + $(TESTS_CFLAGS)
50 +
51 +src_test_network_list_LDADD = \
52 + src/libloc.la
53 +
54 src_test_stringpool_SOURCES = \
55 src/test-stringpool.c
56
57 diff --git a/configure.ac b/configure.ac
58 index 2364dfd..012d8ca 100644
59 --- a/configure.ac
60 +++ b/configure.ac
61 @@ -1,6 +1,6 @@
62 AC_PREREQ(2.60)
63 AC_INIT([libloc],
64 - [0.9.4],
65 + [0.9.5],
66 [location@lists.ipfire.org],
67 [libloc],
68 [https://location.ipfire.org/])
69 diff --git a/src/.gitignore b/src/.gitignore
70 index caf80b5..3ccbdb8 100644
71 --- a/src/.gitignore
72 +++ b/src/.gitignore
73 @@ -10,5 +10,6 @@ test-libloc
74 test-database
75 test-country
76 test-network
77 +test-network-list
78 test-signature
79 test-stringpool
80 diff --git a/src/as-list.c b/src/as-list.c
81 new file mode 100644
82 index 0000000..5acbb8a
83 --- /dev/null
84 +++ b/src/as-list.c
85 @@ -0,0 +1,161 @@
86 +/*
87 + libloc - A library to determine the location of someone on the Internet
88 +
89 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
90 +
91 + This library is free software; you can redistribute it and/or
92 + modify it under the terms of the GNU Lesser General Public
93 + License as published by the Free Software Foundation; either
94 + version 2.1 of the License, or (at your option) any later version.
95 +
96 + This library is distributed in the hope that it will be useful,
97 + but WITHOUT ANY WARRANTY; without even the implied warranty of
98 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
99 + Lesser General Public License for more details.
100 +*/
101 +
102 +#include <errno.h>
103 +#include <stdlib.h>
104 +
105 +#include <loc/as.h>
106 +#include <loc/as-list.h>
107 +#include <loc/private.h>
108 +
109 +struct loc_as_list {
110 + struct loc_ctx* ctx;
111 + int refcount;
112 +
113 + struct loc_as** elements;
114 + size_t elements_size;
115 +
116 + size_t size;
117 +};
118 +
119 +static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
120 + DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
121 + list, size, list->elements_size + size);
122 +
123 + struct loc_as** elements = reallocarray(list->elements,
124 + list->elements_size + size, sizeof(*list->elements));
125 + if (!elements)
126 + return -errno;
127 +
128 + list->elements = elements;
129 + list->elements_size += size;
130 +
131 + return 0;
132 +}
133 +
134 +LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
135 + struct loc_as_list** list) {
136 + struct loc_as_list* l = calloc(1, sizeof(*l));
137 + if (!l)
138 + return -ENOMEM;
139 +
140 + l->ctx = loc_ref(ctx);
141 + l->refcount = 1;
142 +
143 + DEBUG(l->ctx, "AS list allocated at %p\n", l);
144 + *list = l;
145 +
146 + return 0;
147 +}
148 +
149 +LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
150 + list->refcount++;
151 +
152 + return list;
153 +}
154 +
155 +static void loc_as_list_free(struct loc_as_list* list) {
156 + DEBUG(list->ctx, "Releasing AS list at %p\n", list);
157 +
158 + loc_as_list_clear(list);
159 +
160 + loc_unref(list->ctx);
161 + free(list);
162 +}
163 +
164 +LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
165 + if (!list)
166 + return NULL;
167 +
168 + if (--list->refcount > 0)
169 + return list;
170 +
171 + loc_as_list_free(list);
172 + return NULL;
173 +}
174 +
175 +LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
176 + return list->size;
177 +}
178 +
179 +LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
180 + return list->size == 0;
181 +}
182 +
183 +LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
184 + if (!list->elements)
185 + return;
186 +
187 + for (unsigned int i = 0; i < list->size; i++)
188 + loc_as_unref(list->elements[i]);
189 +
190 + free(list->elements);
191 + list->elements = NULL;
192 + list->elements_size = 0;
193 +
194 + list->size = 0;
195 +}
196 +
197 +LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
198 + // Check index
199 + if (index >= list->size)
200 + return NULL;
201 +
202 + return loc_as_ref(list->elements[index]);
203 +}
204 +
205 +LOC_EXPORT int loc_as_list_append(
206 + struct loc_as_list* list, struct loc_as* as) {
207 + if (loc_as_list_contains(list, as))
208 + return 0;
209 +
210 + // Check if we have space left
211 + if (list->size >= list->elements_size) {
212 + int r = loc_as_list_grow(list, 64);
213 + if (r)
214 + return r;
215 + }
216 +
217 + DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
218 +
219 + list->elements[list->size++] = loc_as_ref(as);
220 +
221 + return 0;
222 +}
223 +
224 +LOC_EXPORT int loc_as_list_contains(
225 + struct loc_as_list* list, struct loc_as* as) {
226 + for (unsigned int i = 0; i < list->size; i++) {
227 + if (loc_as_cmp(as, list->elements[i]) == 0)
228 + return 1;
229 + }
230 +
231 + return 0;
232 +}
233 +
234 +LOC_EXPORT int loc_as_list_contains_number(
235 + struct loc_as_list* list, uint32_t number) {
236 + struct loc_as* as;
237 +
238 + int r = loc_as_new(list->ctx, &as, number);
239 + if (r)
240 + return -1;
241 +
242 + r = loc_as_list_contains(list, as);
243 + loc_as_unref(as);
244 +
245 + return r;
246 +}
247 diff --git a/src/as.c b/src/as.c
248 index e1fbb01..757bf3d 100644
249 --- a/src/as.c
250 +++ b/src/as.c
251 @@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
252 }
253
254 LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) {
255 - as->name = strdup(name);
256 + if (as->name)
257 + free(as->name);
258 +
259 + if (name)
260 + as->name = strdup(name);
261 + else
262 + as->name = NULL;
263
264 return 0;
265 }
266 @@ -139,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
267 if (!string)
268 return 1;
269
270 + // Cannot match anything when name is not set
271 + if (!as->name)
272 + return 1;
273 +
274 // Search if string is in name
275 if (strcasestr(as->name, string) != NULL)
276 return 1;
277 diff --git a/src/country-list.c b/src/country-list.c
278 new file mode 100644
279 index 0000000..cc36740
280 --- /dev/null
281 +++ b/src/country-list.c
282 @@ -0,0 +1,161 @@
283 +/*
284 + libloc - A library to determine the location of someone on the Internet
285 +
286 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
287 +
288 + This library is free software; you can redistribute it and/or
289 + modify it under the terms of the GNU Lesser General Public
290 + License as published by the Free Software Foundation; either
291 + version 2.1 of the License, or (at your option) any later version.
292 +
293 + This library is distributed in the hope that it will be useful,
294 + but WITHOUT ANY WARRANTY; without even the implied warranty of
295 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
296 + Lesser General Public License for more details.
297 +*/
298 +
299 +#include <errno.h>
300 +#include <stdlib.h>
301 +
302 +#include <loc/country.h>
303 +#include <loc/country-list.h>
304 +#include <loc/private.h>
305 +
306 +struct loc_country_list {
307 + struct loc_ctx* ctx;
308 + int refcount;
309 +
310 + struct loc_country** elements;
311 + size_t elements_size;
312 +
313 + size_t size;
314 +};
315 +
316 +static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
317 + DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
318 + list, size, list->elements_size + size);
319 +
320 + struct loc_country** elements = reallocarray(list->elements,
321 + list->elements_size + size, sizeof(*list->elements));
322 + if (!elements)
323 + return -errno;
324 +
325 + list->elements = elements;
326 + list->elements_size += size;
327 +
328 + return 0;
329 +}
330 +
331 +LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
332 + struct loc_country_list** list) {
333 + struct loc_country_list* l = calloc(1, sizeof(*l));
334 + if (!l)
335 + return -ENOMEM;
336 +
337 + l->ctx = loc_ref(ctx);
338 + l->refcount = 1;
339 +
340 + DEBUG(l->ctx, "Country list allocated at %p\n", l);
341 + *list = l;
342 +
343 + return 0;
344 +}
345 +
346 +LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
347 + list->refcount++;
348 +
349 + return list;
350 +}
351 +
352 +static void loc_country_list_free(struct loc_country_list* list) {
353 + DEBUG(list->ctx, "Releasing country list at %p\n", list);
354 +
355 + loc_country_list_clear(list);
356 +
357 + loc_unref(list->ctx);
358 + free(list);
359 +}
360 +
361 +LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
362 + if (!list)
363 + return NULL;
364 +
365 + if (--list->refcount > 0)
366 + return list;
367 +
368 + loc_country_list_free(list);
369 + return NULL;
370 +}
371 +
372 +LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
373 + return list->size;
374 +}
375 +
376 +LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
377 + return list->size == 0;
378 +}
379 +
380 +LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
381 + if (!list->elements)
382 + return;
383 +
384 + for (unsigned int i = 0; i < list->size; i++)
385 + loc_country_unref(list->elements[i]);
386 +
387 + free(list->elements);
388 + list->elements = NULL;
389 + list->elements_size = 0;
390 +
391 + list->size = 0;
392 +}
393 +
394 +LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
395 + // Check index
396 + if (index >= list->size)
397 + return NULL;
398 +
399 + return loc_country_ref(list->elements[index]);
400 +}
401 +
402 +LOC_EXPORT int loc_country_list_append(
403 + struct loc_country_list* list, struct loc_country* country) {
404 + if (loc_country_list_contains(list, country))
405 + return 0;
406 +
407 + // Check if we have space left
408 + if (list->size >= list->elements_size) {
409 + int r = loc_country_list_grow(list, 64);
410 + if (r)
411 + return r;
412 + }
413 +
414 + DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
415 +
416 + list->elements[list->size++] = loc_country_ref(country);
417 +
418 + return 0;
419 +}
420 +
421 +LOC_EXPORT int loc_country_list_contains(
422 + struct loc_country_list* list, struct loc_country* country) {
423 + for (unsigned int i = 0; i < list->size; i++) {
424 + if (loc_country_cmp(country, list->elements[i]) == 0)
425 + return 1;
426 + }
427 +
428 + return 0;
429 +}
430 +
431 +LOC_EXPORT int loc_country_list_contains_code(
432 + struct loc_country_list* list, const char* code) {
433 + struct loc_country* country;
434 +
435 + int r = loc_country_new(list->ctx, &country, code);
436 + if (r)
437 + return -1;
438 +
439 + r = loc_country_list_contains(list, country);
440 + loc_country_unref(country);
441 +
442 + return r;
443 +}
444 diff --git a/src/country.c b/src/country.c
445 index 2ba93e6..7aac0db 100644
446 --- a/src/country.c
447 +++ b/src/country.c
448 @@ -34,6 +34,9 @@ struct loc_country {
449 };
450
451 LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
452 + if (!loc_country_code_is_valid(country_code))
453 + return -EINVAL;
454 +
455 struct loc_country* c = calloc(1, sizeof(*c));
456 if (!c)
457 return -ENOMEM;
458 diff --git a/src/database.c b/src/database.c
459 index fa1dad0..1871b74 100644
460 --- a/src/database.c
461 +++ b/src/database.c
462 @@ -38,8 +38,10 @@
463
464 #include <loc/libloc.h>
465 #include <loc/as.h>
466 +#include <loc/as-list.h>
467 #include <loc/compat.h>
468 #include <loc/country.h>
469 +#include <loc/country-list.h>
470 #include <loc/database.h>
471 #include <loc/format.h>
472 #include <loc/network.h>
473 @@ -99,11 +101,14 @@ struct loc_database_enumerator {
474
475 // Search string
476 char* string;
477 - char country_code[3];
478 - uint32_t asn;
479 + struct loc_country_list* countries;
480 + struct loc_as_list* asns;
481 enum loc_network_flags flags;
482 int family;
483
484 + // Flatten output?
485 + int flatten;
486 +
487 // Index of the AS we are looking at
488 unsigned int as_index;
489
490 @@ -115,6 +120,9 @@ struct loc_database_enumerator {
491 struct loc_node_stack network_stack[MAX_STACK_DEPTH];
492 int network_stack_depth;
493 unsigned int* networks_visited;
494 +
495 + // For subnet search
496 + struct loc_network_list* stack;
497 };
498
499 static int loc_database_read_magic(struct loc_database* db) {
500 @@ -611,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
501 }
502
503 clock_t end = clock();
504 - DEBUG(db->ctx, "Signature checked in %.4fms\n",
505 + INFO(db->ctx, "Signature checked in %.4fms\n",
506 (double)(end - start) / CLOCKS_PER_SEC * 1000);
507
508 CLEANUP:
509 @@ -671,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
510 off_t lo = 0;
511 off_t hi = db->as_count - 1;
512
513 +#ifdef ENABLE_DEBUG
514 // Save start time
515 clock_t start = clock();
516 +#endif
517
518 while (lo <= hi) {
519 off_t i = (lo + hi) / 2;
520 @@ -685,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
521 // Check if this is a match
522 uint32_t as_number = loc_as_get_number(*as);
523 if (as_number == number) {
524 +#ifdef ENABLE_DEBUG
525 clock_t end = clock();
526
527 // Log how fast this has been
528 DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
529 (double)(end - start) / CLOCKS_PER_SEC * 1000);
530 +#endif
531
532 return 0;
533 }
534 @@ -733,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
535 return -1;
536 }
537
538 +#ifdef ENABLE_DEBUG
539 if (r == 0) {
540 char* string = loc_network_str(*network);
541 DEBUG(db->ctx, "Got network %s\n", string);
542 free(string);
543 }
544 +#endif
545
546 return r;
547 }
548 @@ -762,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru
549 }
550
551 // Check if the given IP address is inside the network
552 - r = loc_network_match_address(*network, address);
553 - if (r) {
554 + if (!loc_network_match_address(*network, address)) {
555 DEBUG(db->ctx, "Searched address is not part of the network\n");
556
557 loc_network_unref(*network);
558 @@ -832,17 +845,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
559
560 *network = NULL;
561
562 +#ifdef ENABLE_DEBUG
563 // Save start time
564 clock_t start = clock();
565 +#endif
566
567 int r = __loc_database_lookup(db, address, network, &network_address,
568 db->network_nodes_v1, 0);
569
570 +#ifdef ENABLE_DEBUG
571 clock_t end = clock();
572
573 // Log how fast this has been
574 DEBUG(db->ctx, "Executed network search in %.4fms\n",
575 (double)(end - start) / CLOCKS_PER_SEC * 1000);
576 +#endif
577
578 return r;
579 }
580 @@ -889,8 +906,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
581 off_t lo = 0;
582 off_t hi = db->countries_count - 1;
583
584 +#ifdef ENABLE_DEBUG
585 // Save start time
586 clock_t start = clock();
587 +#endif
588
589 while (lo <= hi) {
590 off_t i = (lo + hi) / 2;
591 @@ -905,11 +924,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
592 int result = strcmp(code, cc);
593
594 if (result == 0) {
595 +#ifdef ENABLE_DEBUG
596 clock_t end = clock();
597
598 // Log how fast this has been
599 DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
600 (double)(end - start) / CLOCKS_PER_SEC * 1000);
601 +#endif
602
603 return 0;
604 }
605 @@ -932,8 +953,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
606
607 // Enumerator
608
609 +static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
610 + DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
611 +
612 + // Release all references
613 + loc_database_unref(enumerator->db);
614 + loc_unref(enumerator->ctx);
615 +
616 + if (enumerator->string)
617 + free(enumerator->string);
618 +
619 + if (enumerator->countries)
620 + loc_country_list_unref(enumerator->countries);
621 +
622 + if (enumerator->asns)
623 + loc_as_list_unref(enumerator->asns);
624 +
625 + // Free network search
626 + free(enumerator->networks_visited);
627 +
628 + // Free subnet stack
629 + if (enumerator->stack)
630 + loc_network_list_unref(enumerator->stack);
631 +
632 + free(enumerator);
633 +}
634 +
635 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
636 - struct loc_database* db, enum loc_database_enumerator_mode mode) {
637 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
638 struct loc_database_enumerator* e = calloc(1, sizeof(*e));
639 if (!e)
640 return -ENOMEM;
641 @@ -944,11 +991,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
642 e->mode = mode;
643 e->refcount = 1;
644
645 + // Flatten output?
646 + e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
647 +
648 // Initialise graph search
649 - //e->network_stack[++e->network_stack_depth] = 0;
650 e->network_stack_depth = 1;
651 e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
652
653 + // Allocate stack
654 + int r = loc_network_list_new(e->ctx, &e->stack);
655 + if (r) {
656 + loc_database_enumerator_free(e);
657 + return r;
658 + }
659 +
660 DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
661
662 *enumerator = e;
663 @@ -961,22 +1017,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
664 return enumerator;
665 }
666
667 -static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
668 - DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
669 -
670 - // Release all references
671 - loc_database_unref(enumerator->db);
672 - loc_unref(enumerator->ctx);
673 -
674 - if (enumerator->string)
675 - free(enumerator->string);
676 -
677 - // Free network search
678 - free(enumerator->networks_visited);
679 -
680 - free(enumerator);
681 -}
682 -
683 LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
684 if (!enumerator)
685 return NULL;
686 @@ -998,40 +1038,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
687 return 0;
688 }
689
690 -LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
691 - // Set empty country code
692 - if (!country_code || !*country_code) {
693 - *enumerator->country_code = '\0';
694 - return 0;
695 - }
696 +LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
697 + struct loc_database_enumerator* enumerator) {
698 + if (!enumerator->countries)
699 + return NULL;
700
701 - // Treat A1, A2, A3 as special country codes,
702 - // but perform search for flags instead
703 - if (strcmp(country_code, "A1") == 0) {
704 - return loc_database_enumerator_set_flag(enumerator,
705 - LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
706 - } else if (strcmp(country_code, "A2") == 0) {
707 - return loc_database_enumerator_set_flag(enumerator,
708 - LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
709 - } else if (strcmp(country_code, "A3") == 0) {
710 - return loc_database_enumerator_set_flag(enumerator,
711 - LOC_NETWORK_FLAG_ANYCAST);
712 - }
713 + return loc_country_list_ref(enumerator->countries);
714 +}
715
716 - // Country codes must be two characters
717 - if (!loc_country_code_is_valid(country_code))
718 - return -EINVAL;
719 +LOC_EXPORT int loc_database_enumerator_set_countries(
720 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
721 + if (enumerator->countries)
722 + loc_country_list_unref(enumerator->countries);
723
724 - for (unsigned int i = 0; i < 3; i++) {
725 - enumerator->country_code[i] = country_code[i];
726 - }
727 + enumerator->countries = loc_country_list_ref(countries);
728
729 return 0;
730 }
731
732 -LOC_EXPORT int loc_database_enumerator_set_asn(
733 - struct loc_database_enumerator* enumerator, unsigned int asn) {
734 - enumerator->asn = asn;
735 +LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
736 + struct loc_database_enumerator* enumerator) {
737 + if (!enumerator->asns)
738 + return NULL;
739 +
740 + return loc_as_list_ref(enumerator->asns);
741 +}
742 +
743 +LOC_EXPORT int loc_database_enumerator_set_asns(
744 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
745 + if (enumerator->asns)
746 + loc_as_list_unref(enumerator->asns);
747 +
748 + enumerator->asns = loc_as_list_ref(asns);
749
750 return 0;
751 }
752 @@ -1110,16 +1148,64 @@ static int loc_database_enumerator_stack_push_node(
753 return 0;
754 }
755
756 -LOC_EXPORT int loc_database_enumerator_next_network(
757 - struct loc_database_enumerator* enumerator, struct loc_network** network) {
758 - // Reset network
759 - *network = NULL;
760 +static int loc_database_enumerator_filter_network(
761 + struct loc_database_enumerator* enumerator, struct loc_network* network) {
762 + // Skip if the family does not match
763 + if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
764 + DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
765 + return 1;
766 + }
767
768 - // Do not do anything if not in network mode
769 - if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
770 - return 0;
771 + // Skip if the country code does not match
772 + if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
773 + const char* country_code = loc_network_get_country_code(network);
774
775 - int r;
776 + if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
777 + DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
778 + return 1;
779 + }
780 + }
781 +
782 + // Skip if the ASN does not match
783 + if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
784 + uint32_t asn = loc_network_get_asn(network);
785 +
786 + if (!loc_as_list_contains_number(enumerator->asns, asn)) {
787 + DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
788 + return 1;
789 + }
790 + }
791 +
792 + // Skip if flags do not match
793 + if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
794 + DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
795 + return 1;
796 + }
797 +
798 + // Do not filter
799 + return 0;
800 +}
801 +
802 +static int __loc_database_enumerator_next_network(
803 + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
804 + // Return top element from the stack
805 + while (1) {
806 + *network = loc_network_list_pop_first(enumerator->stack);
807 +
808 + // Stack is empty
809 + if (!*network)
810 + break;
811 +
812 + // Throw away any networks by filter
813 + if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
814 + loc_network_unref(*network);
815 + *network = NULL;
816 + continue;
817 + }
818 +
819 + // Return result
820 + return 0;
821 + }
822
823 DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
824 enumerator->network_stack_depth);
825 @@ -1149,7 +1235,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
826 enumerator->db->network_nodes_v1 + node->offset;
827
828 // Add edges to stack
829 - r = loc_database_enumerator_stack_push_node(enumerator,
830 + int r = loc_database_enumerator_stack_push_node(enumerator,
831 be32toh(n->one), 1, node->depth + 1);
832
833 if (r)
834 @@ -1175,56 +1261,145 @@ LOC_EXPORT int loc_database_enumerator_next_network(
835 if (r)
836 return r;
837
838 - // Check if we are interested in this network
839 + // Return all networks when the filter is disabled
840 + if (!filter)
841 + return 0;
842
843 - // Skip if the family does not match
844 - if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
845 + // Check if we are interested in this network
846 + if (loc_database_enumerator_filter_network(enumerator, *network)) {
847 loc_network_unref(*network);
848 *network = NULL;
849
850 continue;
851 }
852
853 - // Skip if the country code does not match
854 - if (*enumerator->country_code &&
855 - !loc_network_match_country_code(*network, enumerator->country_code)) {
856 - loc_network_unref(*network);
857 - *network = NULL;
858 + return 0;
859 + }
860 + }
861
862 - continue;
863 - }
864 + // Reached the end of the search
865 + return 0;
866 +}
867
868 - // Skip if the ASN does not match
869 - if (enumerator->asn &&
870 - !loc_network_match_asn(*network, enumerator->asn)) {
871 - loc_network_unref(*network);
872 - *network = NULL;
873 +static int __loc_database_enumerator_next_network_flattened(
874 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
875 + // Fetch the next network
876 + int r = __loc_database_enumerator_next_network(enumerator, network, 1);
877 + if (r)
878 + return r;
879
880 - continue;
881 - }
882 + // End if we could not read another network
883 + if (!*network)
884 + return 0;
885
886 - // Skip if flags do not match
887 - if (enumerator->flags &&
888 - !loc_network_match_flag(*network, enumerator->flags)) {
889 - loc_network_unref(*network);
890 - *network = NULL;
891 + struct loc_network* subnet = NULL;
892 + struct loc_network_list* subnets;
893
894 - continue;
895 + // Create a list with all subnets
896 + r = loc_network_list_new(enumerator->ctx, &subnets);
897 + if (r)
898 + return r;
899 +
900 + // Search all subnets from the database
901 + while (1) {
902 + // Fetch the next network in line
903 + r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
904 + if (r) {
905 + loc_network_unref(subnet);
906 + loc_network_list_unref(subnets);
907 +
908 + return r;
909 + }
910 +
911 + // End if we did not receive another subnet
912 + if (!subnet)
913 + break;
914 +
915 + // Collect all subnets in a list
916 + if (loc_network_is_subnet(*network, subnet)) {
917 + r = loc_network_list_push(subnets, subnet);
918 + if (r) {
919 + loc_network_unref(subnet);
920 + loc_network_list_unref(subnets);
921 +
922 + return r;
923 }
924
925 - return 0;
926 + loc_network_unref(subnet);
927 + continue;
928 }
929 +
930 + // If this is not a subnet, we push it back onto the stack and break
931 + r = loc_network_list_push(enumerator->stack, subnet);
932 + if (r) {
933 + loc_network_unref(subnet);
934 + loc_network_list_unref(subnets);
935 +
936 + return r;
937 + }
938 +
939 + loc_network_unref(subnet);
940 + break;
941 }
942
943 - // Reached the end of the search
944 + DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
945 +
946 + // We can abort here if the network has no subnets
947 + if (loc_network_list_empty(subnets)) {
948 + loc_network_list_unref(subnets);
949 +
950 + return 0;
951 + }
952
953 - // Mark all nodes as non-visited
954 - for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
955 - enumerator->networks_visited[i] = 0;
956 + // If the network has any subnets, we will break it into smaller parts
957 + // without the subnets.
958 + struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
959 + if (!excluded) {
960 + loc_network_list_unref(subnets);
961 + return -1;
962 + }
963 +
964 + // Merge subnets onto the stack
965 + r = loc_network_list_merge(enumerator->stack, subnets);
966 + if (r) {
967 + loc_network_list_unref(subnets);
968 + loc_network_list_unref(excluded);
969 +
970 + return r;
971 + }
972 +
973 + // Push excluded list onto the stack
974 + r = loc_network_list_merge(enumerator->stack, excluded);
975 + if (r) {
976 + loc_network_list_unref(subnets);
977 + loc_network_list_unref(excluded);
978 +
979 + return r;
980 + }
981 +
982 + loc_network_list_unref(subnets);
983 + loc_network_list_unref(excluded);
984 +
985 + // Replace network with the first one from the stack
986 + loc_network_unref(*network);
987 + *network = loc_network_list_pop_first(enumerator->stack);
988
989 return 0;
990 }
991
992 +LOC_EXPORT int loc_database_enumerator_next_network(
993 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
994 + // Do not do anything if not in network mode
995 + if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
996 + return 0;
997 +
998 + // Flatten output?
999 + if (enumerator->flatten)
1000 + return __loc_database_enumerator_next_network_flattened(enumerator, network);
1001 +
1002 + return __loc_database_enumerator_next_network(enumerator, network, 1);
1003 +}
1004 +
1005 LOC_EXPORT int loc_database_enumerator_next_country(
1006 struct loc_database_enumerator* enumerator, struct loc_country** country) {
1007 *country = NULL;
1008 diff --git a/src/libloc.sym b/src/libloc.sym
1009 index b8296eb..ee333f1 100644
1010 --- a/src/libloc.sym
1011 +++ b/src/libloc.sym
1012 @@ -37,6 +37,18 @@ global:
1013 loc_as_set_name;
1014 loc_as_unref;
1015
1016 + # AS List
1017 + loc_as_list_append;
1018 + loc_as_list_clear;
1019 + loc_as_list_contains;
1020 + loc_as_list_contains_number;
1021 + loc_as_list_empty;
1022 + loc_as_list_get;
1023 + loc_as_list_new;
1024 + loc_as_list_ref;
1025 + loc_as_list_size;
1026 + loc_as_list_unref;
1027 +
1028 # Country
1029 loc_country_cmp;
1030 loc_country_code_is_valid;
1031 @@ -49,6 +61,18 @@ global:
1032 loc_country_set_name;
1033 loc_country_unref;
1034
1035 + # Country List
1036 + loc_country_list_append;
1037 + loc_country_list_clear;
1038 + loc_country_list_contains;
1039 + loc_country_list_contains_code;
1040 + loc_country_list_empty;
1041 + loc_country_list_get;
1042 + loc_country_list_new;
1043 + loc_country_list_ref;
1044 + loc_country_list_size;
1045 + loc_country_list_unref;
1046 +
1047 # Database
1048 loc_database_add_as;
1049 loc_database_count_as;
1050 @@ -66,13 +90,15 @@ global:
1051 loc_database_verify;
1052
1053 # Database Enumerator
1054 + loc_database_enumerator_get_asns;
1055 + loc_database_enumerator_get_countries;
1056 loc_database_enumerator_new;
1057 loc_database_enumerator_next_as;
1058 loc_database_enumerator_next_country;
1059 loc_database_enumerator_next_network;
1060 loc_database_enumerator_ref;
1061 - loc_database_enumerator_set_asn;
1062 - loc_database_enumerator_set_country_code;
1063 + loc_database_enumerator_set_asns;
1064 + loc_database_enumerator_set_countries;
1065 loc_database_enumerator_set_family;
1066 loc_database_enumerator_set_flag;
1067 loc_database_enumerator_set_string;
1068 @@ -80,24 +106,48 @@ global:
1069
1070 # Network
1071 loc_network_address_family;
1072 + loc_network_cmp;
1073 + loc_network_exclude;
1074 + loc_network_exclude_list;
1075 loc_network_format_first_address;
1076 loc_network_format_last_address;
1077 loc_network_get_asn;
1078 loc_network_get_country_code;
1079 + loc_network_get_first_address;
1080 + loc_network_get_last_address;
1081 loc_network_has_flag;
1082 - loc_network_is_subnet_of;
1083 + loc_network_is_subnet;
1084 + loc_network_match_address;
1085 loc_network_match_asn;
1086 loc_network_match_country_code;
1087 loc_network_match_flag;
1088 loc_network_new;
1089 loc_network_new_from_string;
1090 + loc_network_overlaps;
1091 + loc_network_prefix;
1092 loc_network_ref;
1093 loc_network_set_asn;
1094 loc_network_set_country_code;
1095 loc_network_set_flag;
1096 loc_network_str;
1097 + loc_network_subnets;
1098 loc_network_unref;
1099
1100 + # Network List
1101 + loc_network_list_clear;
1102 + loc_network_list_contains;
1103 + loc_network_list_dump;
1104 + loc_network_list_empty;
1105 + loc_network_list_get;
1106 + loc_network_list_merge;
1107 + loc_network_list_new;
1108 + loc_network_list_pop;
1109 + loc_network_list_pop_first;
1110 + loc_network_list_push;
1111 + loc_network_list_ref;
1112 + loc_network_list_size;
1113 + loc_network_list_unref;
1114 +
1115 # Writer
1116 loc_writer_add_as;
1117 loc_writer_add_country;
1118 diff --git a/src/loc/as-list.h b/src/loc/as-list.h
1119 new file mode 100644
1120 index 0000000..7b5c4e8
1121 --- /dev/null
1122 +++ b/src/loc/as-list.h
1123 @@ -0,0 +1,41 @@
1124 +/*
1125 + libloc - A library to determine the location of someone on the Internet
1126 +
1127 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
1128 +
1129 + This library is free software; you can redistribute it and/or
1130 + modify it under the terms of the GNU Lesser General Public
1131 + License as published by the Free Software Foundation; either
1132 + version 2.1 of the License, or (at your option) any later version.
1133 +
1134 + This library is distributed in the hope that it will be useful,
1135 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1136 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1137 + Lesser General Public License for more details.
1138 +*/
1139 +
1140 +#ifndef LIBLOC_AS_LIST_H
1141 +#define LIBLOC_AS_LIST_H
1142 +
1143 +#include <loc/as.h>
1144 +#include <loc/libloc.h>
1145 +
1146 +struct loc_as_list;
1147 +
1148 +int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
1149 +struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
1150 +struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
1151 +
1152 +size_t loc_as_list_size(struct loc_as_list* list);
1153 +int loc_as_list_empty(struct loc_as_list* list);
1154 +void loc_as_list_clear(struct loc_as_list* list);
1155 +
1156 +struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
1157 +int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
1158 +
1159 +int loc_as_list_contains(
1160 + struct loc_as_list* list, struct loc_as* as);
1161 +int loc_as_list_contains_number(
1162 + struct loc_as_list* list, uint32_t number);
1163 +
1164 +#endif
1165 diff --git a/src/loc/country-list.h b/src/loc/country-list.h
1166 new file mode 100644
1167 index 0000000..a7f818a
1168 --- /dev/null
1169 +++ b/src/loc/country-list.h
1170 @@ -0,0 +1,43 @@
1171 +/*
1172 + libloc - A library to determine the location of someone on the Internet
1173 +
1174 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
1175 +
1176 + This library is free software; you can redistribute it and/or
1177 + modify it under the terms of the GNU Lesser General Public
1178 + License as published by the Free Software Foundation; either
1179 + version 2.1 of the License, or (at your option) any later version.
1180 +
1181 + This library is distributed in the hope that it will be useful,
1182 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1183 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1184 + Lesser General Public License for more details.
1185 +*/
1186 +
1187 +#ifndef LIBLOC_COUNTRY_LIST_H
1188 +#define LIBLOC_COUNTRY_LIST_H
1189 +
1190 +#include <stdlib.h>
1191 +
1192 +#include <loc/libloc.h>
1193 +#include <loc/country.h>
1194 +
1195 +struct loc_country_list;
1196 +
1197 +int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
1198 +struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
1199 +struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
1200 +
1201 +size_t loc_country_list_size(struct loc_country_list* list);
1202 +int loc_country_list_empty(struct loc_country_list* list);
1203 +void loc_country_list_clear(struct loc_country_list* list);
1204 +
1205 +struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
1206 +int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
1207 +
1208 +int loc_country_list_contains(
1209 + struct loc_country_list* list, struct loc_country* country);
1210 +int loc_country_list_contains_code(
1211 + struct loc_country_list* list, const char* code);
1212 +
1213 +#endif
1214 diff --git a/src/loc/database.h b/src/loc/database.h
1215 index 43173dd..70801f0 100644
1216 --- a/src/loc/database.h
1217 +++ b/src/loc/database.h
1218 @@ -25,6 +25,7 @@
1219 #include <loc/network.h>
1220 #include <loc/as.h>
1221 #include <loc/country.h>
1222 +#include <loc/country-list.h>
1223
1224 struct loc_database;
1225 int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
1226 @@ -55,15 +56,24 @@ enum loc_database_enumerator_mode {
1227 LOC_DB_ENUMERATE_COUNTRIES = 3,
1228 };
1229
1230 +enum loc_database_enumerator_flags {
1231 + LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
1232 +};
1233 +
1234 struct loc_database_enumerator;
1235 int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
1236 - struct loc_database* db, enum loc_database_enumerator_mode mode);
1237 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
1238 struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
1239 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
1240
1241 int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
1242 -int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
1243 -int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
1244 +struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
1245 +int loc_database_enumerator_set_countries(
1246 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
1247 +struct loc_as_list* loc_database_enumerator_get_asns(
1248 + struct loc_database_enumerator* enumerator);
1249 +int loc_database_enumerator_set_asns(
1250 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
1251 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
1252 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
1253 int loc_database_enumerator_next_as(
1254 diff --git a/src/loc/network-list.h b/src/loc/network-list.h
1255 new file mode 100644
1256 index 0000000..bee21c4
1257 --- /dev/null
1258 +++ b/src/loc/network-list.h
1259 @@ -0,0 +1,37 @@
1260 +/*
1261 + libloc - A library to determine the location of someone on the Internet
1262 +
1263 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
1264 +
1265 + This library is free software; you can redistribute it and/or
1266 + modify it under the terms of the GNU Lesser General Public
1267 + License as published by the Free Software Foundation; either
1268 + version 2.1 of the License, or (at your option) any later version.
1269 +
1270 + This library is distributed in the hope that it will be useful,
1271 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1272 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1273 + Lesser General Public License for more details.
1274 +*/
1275 +
1276 +#ifndef LIBLOC_NETWORK_LIST_H
1277 +#define LIBLOC_NETWORK_LIST_H
1278 +
1279 +#include <loc/network.h>
1280 +
1281 +struct loc_network_list;
1282 +int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
1283 +struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
1284 +struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
1285 +size_t loc_network_list_size(struct loc_network_list* list);
1286 +int loc_network_list_empty(struct loc_network_list* list);
1287 +void loc_network_list_clear(struct loc_network_list* list);
1288 +void loc_network_list_dump(struct loc_network_list* list);
1289 +struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
1290 +int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
1291 +struct loc_network* loc_network_list_pop(struct loc_network_list* list);
1292 +struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
1293 +int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
1294 +int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
1295 +
1296 +#endif
1297 diff --git a/src/loc/network.h b/src/loc/network.h
1298 index 70c3803..af3dafd 100644
1299 --- a/src/loc/network.h
1300 +++ b/src/loc/network.h
1301 @@ -21,6 +21,7 @@
1302
1303 #include <loc/libloc.h>
1304 #include <loc/format.h>
1305 +#include <loc/network-list.h>
1306
1307 enum loc_network_flags {
1308 LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
1309 @@ -37,8 +38,11 @@ struct loc_network* loc_network_ref(struct loc_network* network);
1310 struct loc_network* loc_network_unref(struct loc_network* network);
1311 char* loc_network_str(struct loc_network* network);
1312 int loc_network_address_family(struct loc_network* network);
1313 +unsigned int loc_network_prefix(struct loc_network* network);
1314
1315 +const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
1316 char* loc_network_format_first_address(struct loc_network* network);
1317 +const struct in6_addr* loc_network_get_last_address(struct loc_network* network);
1318 char* loc_network_format_last_address(struct loc_network* network);
1319 int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
1320
1321 @@ -54,7 +58,14 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
1322 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
1323 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
1324
1325 -int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
1326 +int loc_network_cmp(struct loc_network* self, struct loc_network* other);
1327 +int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
1328 +int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
1329 +int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
1330 +struct loc_network_list* loc_network_exclude(
1331 + struct loc_network* self, struct loc_network* other);
1332 +struct loc_network_list* loc_network_exclude_list(
1333 + struct loc_network* network, struct loc_network_list* list);
1334
1335 #ifdef LIBLOC_PRIVATE
1336
1337 diff --git a/src/network-list.c b/src/network-list.c
1338 new file mode 100644
1339 index 0000000..e2b20f3
1340 --- /dev/null
1341 +++ b/src/network-list.c
1342 @@ -0,0 +1,295 @@
1343 +/*
1344 + libloc - A library to determine the location of someone on the Internet
1345 +
1346 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
1347 +
1348 + This library is free software; you can redistribute it and/or
1349 + modify it under the terms of the GNU Lesser General Public
1350 + License as published by the Free Software Foundation; either
1351 + version 2.1 of the License, or (at your option) any later version.
1352 +
1353 + This library is distributed in the hope that it will be useful,
1354 + but WITHOUT ANY WARRANTY; without even the implied warranty of
1355 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1356 + Lesser General Public License for more details.
1357 +*/
1358 +
1359 +#include <errno.h>
1360 +#include <stdlib.h>
1361 +#include <time.h>
1362 +
1363 +#include <loc/libloc.h>
1364 +#include <loc/network.h>
1365 +#include <loc/private.h>
1366 +
1367 +struct loc_network_list {
1368 + struct loc_ctx* ctx;
1369 + int refcount;
1370 +
1371 + struct loc_network** elements;
1372 + size_t elements_size;
1373 +
1374 + size_t size;
1375 +};
1376 +
1377 +static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
1378 + DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
1379 + list, size, list->elements_size + size);
1380 +
1381 + struct loc_network** elements = reallocarray(list->elements,
1382 + list->elements_size + size, sizeof(*list->elements));
1383 + if (!elements)
1384 + return -errno;
1385 +
1386 + list->elements = elements;
1387 + list->elements_size += size;
1388 +
1389 + return 0;
1390 +}
1391 +
1392 +LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
1393 + struct loc_network_list** list) {
1394 + struct loc_network_list* l = calloc(1, sizeof(*l));
1395 + if (!l)
1396 + return -ENOMEM;
1397 +
1398 + l->ctx = loc_ref(ctx);
1399 + l->refcount = 1;
1400 +
1401 + DEBUG(l->ctx, "Network list allocated at %p\n", l);
1402 + *list = l;
1403 + return 0;
1404 +}
1405 +
1406 +LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
1407 + list->refcount++;
1408 +
1409 + return list;
1410 +}
1411 +
1412 +static void loc_network_list_free(struct loc_network_list* list) {
1413 + DEBUG(list->ctx, "Releasing network list at %p\n", list);
1414 +
1415 + // Remove all content
1416 + loc_network_list_clear(list);
1417 +
1418 + loc_unref(list->ctx);
1419 + free(list);
1420 +}
1421 +
1422 +LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
1423 + if (!list)
1424 + return NULL;
1425 +
1426 + if (--list->refcount > 0)
1427 + return list;
1428 +
1429 + loc_network_list_free(list);
1430 + return NULL;
1431 +}
1432 +
1433 +LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
1434 + return list->size;
1435 +}
1436 +
1437 +LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
1438 + return list->size == 0;
1439 +}
1440 +
1441 +LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
1442 + if (!list->elements)
1443 + return;
1444 +
1445 + for (unsigned int i = 0; i < list->size; i++)
1446 + loc_network_unref(list->elements[i]);
1447 +
1448 + free(list->elements);
1449 + list->elements = NULL;
1450 + list->elements_size = 0;
1451 +
1452 + list->size = 0;
1453 +}
1454 +
1455 +LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
1456 + struct loc_network* network;
1457 + char* s;
1458 +
1459 + for (unsigned int i = 0; i < list->size; i++) {
1460 + network = list->elements[i];
1461 +
1462 + s = loc_network_str(network);
1463 +
1464 + INFO(list->ctx, "%4d: %s\n", i, s);
1465 + free(s);
1466 + }
1467 +}
1468 +
1469 +LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
1470 + // Check index
1471 + if (index >= list->size)
1472 + return NULL;
1473 +
1474 + return loc_network_ref(list->elements[index]);
1475 +}
1476 +
1477 +static off_t loc_network_list_find(struct loc_network_list* list,
1478 + struct loc_network* network, int* found) {
1479 + off_t lo = 0;
1480 + off_t hi = list->size - 1;
1481 + int result;
1482 +
1483 + // Since we are working on an ordered list, there is often a good chance that
1484 + // the network we are looking for is at the end or has to go to the end.
1485 + if (hi >= 0) {
1486 + result = loc_network_cmp(network, list->elements[hi]);
1487 +
1488 + // Match, so we are done
1489 + if (result == 0) {
1490 + *found = 1;
1491 +
1492 + return hi;
1493 +
1494 + // This needs to be added after the last one
1495 + } else if (result > 0) {
1496 + *found = 0;
1497 +
1498 + return hi + 1;
1499 + }
1500 + }
1501 +
1502 +#ifdef ENABLE_DEBUG
1503 + // Save start time
1504 + clock_t start = clock();
1505 +#endif
1506 +
1507 + off_t i = 0;
1508 +
1509 + while (lo <= hi) {
1510 + i = (lo + hi) / 2;
1511 +
1512 + // Check if this is a match
1513 + result = loc_network_cmp(network, list->elements[i]);
1514 +
1515 + if (result == 0) {
1516 + *found = 1;
1517 +
1518 +#ifdef ENABLE_DEBUG
1519 + clock_t end = clock();
1520 +
1521 + // Log how fast this has been
1522 + DEBUG(list->ctx, "Found network in %.4fms at %jd\n",
1523 + (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
1524 +#endif
1525 +
1526 + return i;
1527 + }
1528 +
1529 + if (result > 0) {
1530 + lo = i + 1;
1531 + i++;
1532 + } else {
1533 + hi = i - 1;
1534 + }
1535 + }
1536 +
1537 + *found = 0;
1538 +
1539 +#ifdef ENABLE_DEBUG
1540 + clock_t end = clock();
1541 +
1542 + // Log how fast this has been
1543 + DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n",
1544 + (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
1545 +#endif
1546 +
1547 + return i;
1548 +}
1549 +
1550 +LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
1551 + int found = 0;
1552 +
1553 + off_t index = loc_network_list_find(list, network, &found);
1554 +
1555 + // The network has been found on the list. Nothing to do.
1556 + if (found)
1557 + return 0;
1558 +
1559 + DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n",
1560 + list, network, (intmax_t)index);
1561 +
1562 + // Check if we have space left
1563 + if (list->size >= list->elements_size) {
1564 + int r = loc_network_list_grow(list, 64);
1565 + if (r)
1566 + return r;
1567 + }
1568 +
1569 + // The list is now larger
1570 + list->size++;
1571 +
1572 + // Move all elements out of the way
1573 + for (unsigned int i = list->size - 1; i > index; i--)
1574 + list->elements[i] = list->elements[i - 1];
1575 +
1576 + // Add the new element at the right place
1577 + list->elements[index] = loc_network_ref(network);
1578 +
1579 + return 0;
1580 +}
1581 +
1582 +LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
1583 + // Return nothing when empty
1584 + if (loc_network_list_empty(list)) {
1585 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
1586 + return NULL;
1587 + }
1588 +
1589 + struct loc_network* network = list->elements[--list->size];
1590 +
1591 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
1592 +
1593 + return network;
1594 +}
1595 +
1596 +LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
1597 + // Return nothing when empty
1598 + if (loc_network_list_empty(list)) {
1599 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
1600 + return NULL;
1601 + }
1602 +
1603 + struct loc_network* network = list->elements[0];
1604 +
1605 + // Move all elements to the top of the stack
1606 + for (unsigned int i = 0; i < list->size - 1; i++) {
1607 + list->elements[i] = list->elements[i+1];
1608 + }
1609 +
1610 + // The list is shorter now
1611 + --list->size;
1612 +
1613 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
1614 +
1615 + return network;
1616 +}
1617 +
1618 +LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
1619 + int found = 0;
1620 +
1621 + loc_network_list_find(list, network, &found);
1622 +
1623 + return found;
1624 +}
1625 +
1626 +LOC_EXPORT int loc_network_list_merge(
1627 + struct loc_network_list* self, struct loc_network_list* other) {
1628 + int r;
1629 +
1630 + for (unsigned int i = 0; i < other->size; i++) {
1631 + r = loc_network_list_push(self, other->elements[i]);
1632 + if (r)
1633 + return r;
1634 + }
1635 +
1636 + return 0;
1637 +}
1638 diff --git a/src/network.c b/src/network.c
1639 index 366caa2..a6b679c 100644
1640 --- a/src/network.c
1641 +++ b/src/network.c
1642 @@ -29,6 +29,7 @@
1643 #include <loc/compat.h>
1644 #include <loc/country.h>
1645 #include <loc/network.h>
1646 +#include <loc/network-list.h>
1647 #include <loc/private.h>
1648
1649 struct loc_network {
1650 @@ -97,6 +98,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
1651 return a;
1652 }
1653
1654 +static struct in6_addr address_increment(const struct in6_addr* address) {
1655 + struct in6_addr a = *address;
1656 +
1657 + for (int octet = 15; octet >= 0; octet--) {
1658 + if (a.s6_addr[octet] < 255) {
1659 + a.s6_addr[octet]++;
1660 + break;
1661 + } else {
1662 + a.s6_addr[octet] = 0;
1663 + }
1664 + }
1665 +
1666 + return a;
1667 +}
1668 +
1669 LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
1670 struct in6_addr* address, unsigned int prefix) {
1671 // Address cannot be unspecified
1672 @@ -160,9 +176,11 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
1673 LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
1674 const char* address_string) {
1675 struct in6_addr first_address;
1676 - unsigned int prefix = 0;
1677 char* prefix_string;
1678 - int r = 1;
1679 + unsigned int prefix = 128;
1680 + int r = -EINVAL;
1681 +
1682 + DEBUG(ctx, "Attempting to parse network %s\n", address_string);
1683
1684 // Make a copy of the string to work on it
1685 char* buffer = strdup(address_string);
1686 @@ -171,29 +189,40 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1687 // Split address and prefix
1688 address_string = strsep(&prefix_string, "/");
1689
1690 - // Did we find a prefix?
1691 + DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
1692 +
1693 + // Parse the address
1694 + r = loc_parse_address(ctx, address_string, &first_address);
1695 + if (r) {
1696 + DEBUG(ctx, "The address could not be parsed\n");
1697 + goto FAIL;
1698 + }
1699 +
1700 + // If a prefix was given, we will try to parse it
1701 if (prefix_string) {
1702 // Convert prefix to integer
1703 prefix = strtol(prefix_string, NULL, 10);
1704
1705 - if (prefix) {
1706 - // Parse the address
1707 - r = loc_parse_address(ctx, address_string, &first_address);
1708 -
1709 - // Map the prefix to IPv6 if needed
1710 - if (IN6_IS_ADDR_V4MAPPED(&first_address))
1711 - prefix += 96;
1712 + if (!prefix) {
1713 + DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
1714 + goto FAIL;
1715 }
1716 +
1717 + // Map the prefix to IPv6 if needed
1718 + if (IN6_IS_ADDR_V4MAPPED(&first_address))
1719 + prefix += 96;
1720 }
1721
1722 +FAIL:
1723 // Free temporary buffer
1724 free(buffer);
1725
1726 - if (r == 0) {
1727 - r = loc_network_new(ctx, network, &first_address, prefix);
1728 - }
1729 + // Exit if the parsing was unsuccessful
1730 + if (r)
1731 + return r;
1732
1733 - return r;
1734 + // Create a new network
1735 + return loc_network_new(ctx, network, &first_address, prefix);
1736 }
1737
1738 LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
1739 @@ -281,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) {
1740 return network->family;
1741 }
1742
1743 +LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) {
1744 + switch (network->family) {
1745 + case AF_INET6:
1746 + return network->prefix;
1747 +
1748 + case AF_INET:
1749 + return network->prefix - 96;
1750 + }
1751 +
1752 + return 0;
1753 +}
1754 +
1755 static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) {
1756 const size_t length = INET6_ADDRSTRLEN;
1757
1758 @@ -314,10 +355,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc
1759 return string;
1760 }
1761
1762 +LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) {
1763 + return &network->first_address;
1764 +}
1765 +
1766 LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) {
1767 return loc_network_format_address(network, &network->first_address);
1768 }
1769
1770 +LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) {
1771 + return &network->last_address;
1772 +}
1773 +
1774 LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
1775 return loc_network_format_address(network, &network->last_address);
1776 }
1777 @@ -325,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
1778 LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) {
1779 // Address must be larger than the start address
1780 if (in6_addr_cmp(&network->first_address, address) > 0)
1781 - return 1;
1782 + return 0;
1783
1784 // Address must be smaller than the last address
1785 if (in6_addr_cmp(&network->last_address, address) < 0)
1786 - return 1;
1787 + return 0;
1788
1789 // The address is inside this network
1790 - return 0;
1791 + return 1;
1792 }
1793
1794 LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) {
1795 @@ -392,20 +441,310 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
1796 return loc_network_has_flag(network, flag);
1797 }
1798
1799 -LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
1800 +LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
1801 + // Compare address
1802 + int r = in6_addr_cmp(&self->first_address, &other->first_address);
1803 + if (r)
1804 + return r;
1805 +
1806 + // Compare prefix
1807 + if (self->prefix > other->prefix)
1808 + return 1;
1809 + else if (self->prefix < other->prefix)
1810 + return -1;
1811 +
1812 + // Both networks are equal
1813 + return 0;
1814 +}
1815 +
1816 +LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
1817 + // Either of the start addresses must be in the other subnet
1818 + if (loc_network_match_address(self, &other->first_address))
1819 + return 1;
1820 +
1821 + if (loc_network_match_address(other, &self->first_address))
1822 + return 1;
1823 +
1824 + // Or either of the end addresses is in the other subnet
1825 + if (loc_network_match_address(self, &other->last_address))
1826 + return 1;
1827 +
1828 + if (loc_network_match_address(other, &self->last_address))
1829 + return 1;
1830 +
1831 + return 0;
1832 +}
1833 +
1834 +LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
1835 + // The prefix must be smaller (this avoids the more complex comparisons later)
1836 + if (self->prefix > other->prefix)
1837 + return 0;
1838 +
1839 // If the start address of the other network is smaller than this network,
1840 // it cannot be a subnet.
1841 - if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
1842 + if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
1843 return 0;
1844
1845 // If the end address of the other network is greater than this network,
1846 // it cannot be a subnet.
1847 - if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
1848 + if (in6_addr_cmp(&self->last_address, &other->last_address) < 0)
1849 return 0;
1850
1851 return 1;
1852 }
1853
1854 +LOC_EXPORT int loc_network_subnets(struct loc_network* network,
1855 + struct loc_network** subnet1, struct loc_network** subnet2) {
1856 + int r;
1857 + *subnet1 = NULL;
1858 + *subnet2 = NULL;
1859 +
1860 + // New prefix length
1861 + unsigned int prefix = network->prefix + 1;
1862 +
1863 + // Check if the new prefix is valid
1864 + if (valid_prefix(&network->first_address, prefix))
1865 + return -1;
1866 +
1867 + // Create the first half of the network
1868 + r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix);
1869 + if (r)
1870 + return r;
1871 +
1872 + // The next subnet starts after the first one
1873 + struct in6_addr first_address = address_increment(&(*subnet1)->last_address);
1874 +
1875 + // Create the second half of the network
1876 + r = loc_network_new(network->ctx, subnet2, &first_address, prefix);
1877 + if (r)
1878 + return r;
1879 +
1880 + // Copy country code
1881 + const char* country_code = loc_network_get_country_code(network);
1882 + if (country_code) {
1883 + loc_network_set_country_code(*subnet1, country_code);
1884 + loc_network_set_country_code(*subnet2, country_code);
1885 + }
1886 +
1887 + // Copy ASN
1888 + uint32_t asn = loc_network_get_asn(network);
1889 + if (asn) {
1890 + loc_network_set_asn(*subnet1, asn);
1891 + loc_network_set_asn(*subnet2, asn);
1892 + }
1893 +
1894 + // Copy flags
1895 + loc_network_set_flag(*subnet1, network->flags);
1896 + loc_network_set_flag(*subnet2, network->flags);
1897 +
1898 + return 0;
1899 +}
1900 +
1901 +static int __loc_network_exclude(struct loc_network* network,
1902 + struct loc_network* other, struct loc_network_list* list) {
1903 + struct loc_network* subnet1 = NULL;
1904 + struct loc_network* subnet2 = NULL;
1905 +
1906 + int r = loc_network_subnets(network, &subnet1, &subnet2);
1907 + if (r)
1908 + goto ERROR;
1909 +
1910 + if (loc_network_cmp(other, subnet1) == 0) {
1911 + r = loc_network_list_push(list, subnet2);
1912 + if (r)
1913 + goto ERROR;
1914 +
1915 + } else if (loc_network_cmp(other, subnet2) == 0) {
1916 + r = loc_network_list_push(list, subnet1);
1917 + if (r)
1918 + goto ERROR;
1919 +
1920 + } else if (loc_network_is_subnet(subnet1, other)) {
1921 + r = loc_network_list_push(list, subnet2);
1922 + if (r)
1923 + goto ERROR;
1924 +
1925 + r = __loc_network_exclude(subnet1, other, list);
1926 + if (r)
1927 + goto ERROR;
1928 +
1929 + } else if (loc_network_is_subnet(subnet2, other)) {
1930 + r = loc_network_list_push(list, subnet1);
1931 + if (r)
1932 + goto ERROR;
1933 +
1934 + r = __loc_network_exclude(subnet2, other, list);
1935 + if (r)
1936 + goto ERROR;
1937 +
1938 + } else {
1939 + ERROR(network->ctx, "We should never get here\n");
1940 + r = 1;
1941 + goto ERROR;
1942 + }
1943 +
1944 +ERROR:
1945 + if (subnet1)
1946 + loc_network_unref(subnet1);
1947 +
1948 + if (subnet2)
1949 + loc_network_unref(subnet2);
1950 +
1951 + return r;
1952 +}
1953 +
1954 +static int __loc_network_exclude_to_list(struct loc_network* self,
1955 + struct loc_network* other, struct loc_network_list* list) {
1956 + // Other must be a subnet of self
1957 + if (!loc_network_is_subnet(self, other)) {
1958 + DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
1959 +
1960 + // Exit silently
1961 + return 0;
1962 + }
1963 +
1964 + // We cannot perform this operation if both networks equal
1965 + if (loc_network_cmp(self, other) == 0) {
1966 + DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
1967 +
1968 + // Exit silently
1969 + return 0;
1970 + }
1971 +
1972 + return __loc_network_exclude(self, other, list);
1973 +}
1974 +
1975 +LOC_EXPORT struct loc_network_list* loc_network_exclude(
1976 + struct loc_network* self, struct loc_network* other) {
1977 + struct loc_network_list* list;
1978 +
1979 +#ifdef ENABLE_DEBUG
1980 + char* n1 = loc_network_str(self);
1981 + char* n2 = loc_network_str(other);
1982 +
1983 + DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
1984 +
1985 + free(n1);
1986 + free(n2);
1987 +#endif
1988 +
1989 + // Create a new list with the result
1990 + int r = loc_network_list_new(self->ctx, &list);
1991 + if (r) {
1992 + ERROR(self->ctx, "Could not create network list: %d\n", r);
1993 +
1994 + return NULL;
1995 + }
1996 +
1997 + r = __loc_network_exclude_to_list(self, other, list);
1998 + if (r) {
1999 + loc_network_list_unref(list);
2000 +
2001 + return NULL;
2002 + }
2003 +
2004 + // Return the result
2005 + return list;
2006 +}
2007 +
2008 +LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
2009 + struct loc_network* network, struct loc_network_list* list) {
2010 + struct loc_network_list* to_check;
2011 +
2012 + // Create a new list with all networks to look at
2013 + int r = loc_network_list_new(network->ctx, &to_check);
2014 + if (r)
2015 + return NULL;
2016 +
2017 + struct loc_network* subnet = NULL;
2018 + struct loc_network_list* subnets = NULL;
2019 +
2020 + for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
2021 + subnet = loc_network_list_get(list, i);
2022 +
2023 + // Find all excluded networks
2024 + if (!loc_network_list_contains(to_check, subnet)) {
2025 + r = __loc_network_exclude_to_list(network, subnet, to_check);
2026 + if (r) {
2027 + loc_network_list_unref(to_check);
2028 + loc_network_unref(subnet);
2029 +
2030 + return NULL;
2031 + }
2032 + }
2033 +
2034 + // Cleanup
2035 + loc_network_unref(subnet);
2036 + }
2037 +
2038 + r = loc_network_list_new(network->ctx, &subnets);
2039 + if (r) {
2040 + loc_network_list_unref(to_check);
2041 + return NULL;
2042 + }
2043 +
2044 + off_t smallest_subnet = 0;
2045 +
2046 + while (!loc_network_list_empty(to_check)) {
2047 + struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check);
2048 +
2049 + // Check whether the subnet to check is part of the input list
2050 + if (loc_network_list_contains(list, subnet_to_check)) {
2051 + loc_network_unref(subnet_to_check);
2052 + continue;
2053 + }
2054 +
2055 + // Marks whether this subnet passed all checks
2056 + int passed = 1;
2057 +
2058 + for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) {
2059 + subnet = loc_network_list_get(list, i);
2060 +
2061 + // Drop this subnet if is a subnet of another subnet
2062 + if (loc_network_is_subnet(subnet, subnet_to_check)) {
2063 + passed = 0;
2064 + loc_network_unref(subnet);
2065 + break;
2066 + }
2067 +
2068 + // Break it down if it overlaps
2069 + if (loc_network_overlaps(subnet, subnet_to_check)) {
2070 + passed = 0;
2071 +
2072 + __loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
2073 +
2074 + loc_network_unref(subnet);
2075 + break;
2076 + }
2077 +
2078 + // If the subnet is strictly greater, we do not need to continue the search
2079 + r = loc_network_cmp(subnet, subnet_to_check);
2080 + if (r > 0) {
2081 + loc_network_unref(subnet);
2082 + break;
2083 +
2084 + // If it is strictly smaller, we can continue the search from here next
2085 + // time because all networks that are to be checked can only be larger
2086 + // than this one.
2087 + } else if (r < 0) {
2088 + smallest_subnet = i;
2089 + }
2090 +
2091 + loc_network_unref(subnet);
2092 + }
2093 +
2094 + if (passed) {
2095 + r = loc_network_list_push(subnets, subnet_to_check);
2096 + }
2097 +
2098 + loc_network_unref(subnet_to_check);
2099 + }
2100 +
2101 + loc_network_list_unref(to_check);
2102 +
2103 + return subnets;
2104 +}
2105 +
2106 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
2107 // Add country code
2108 loc_country_code_copy(dbobj->country_code, network->country_code);
2109 @@ -474,7 +813,7 @@ struct loc_network_tree_node {
2110 struct loc_network* network;
2111 };
2112
2113 -LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
2114 +int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
2115 struct loc_network_tree* t = calloc(1, sizeof(*t));
2116 if (!t)
2117 return -ENOMEM;
2118 @@ -494,7 +833,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
2119 return 0;
2120 }
2121
2122 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
2123 +struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
2124 return loc_network_tree_node_ref(tree->root);
2125 }
2126
2127 @@ -566,7 +905,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
2128 return 0;
2129 }
2130
2131 -LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
2132 +int loc_network_tree_walk(struct loc_network_tree* tree,
2133 int(*filter_callback)(struct loc_network* network, void* data),
2134 int(*callback)(struct loc_network* network, void* data), void* data) {
2135 return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
2136 @@ -581,7 +920,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
2137 free(tree);
2138 }
2139
2140 -LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
2141 +struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
2142 if (--tree->refcount > 0)
2143 return tree;
2144
2145 @@ -602,13 +941,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
2146 return 0;
2147 }
2148
2149 -LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
2150 +int loc_network_tree_dump(struct loc_network_tree* tree) {
2151 DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
2152
2153 return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
2154 }
2155
2156 -LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
2157 +int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
2158 DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
2159
2160 struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
2161 @@ -639,7 +978,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
2162 return 0;
2163 }
2164
2165 -LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
2166 +size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
2167 size_t counter = 0;
2168
2169 int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
2170 @@ -661,11 +1000,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
2171 return counter;
2172 }
2173
2174 -LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
2175 +size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
2176 return __loc_network_tree_count_nodes(tree->root);
2177 }
2178
2179 -LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
2180 +int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
2181 struct loc_network_tree_node* n = calloc(1, sizeof(*n));
2182 if (!n)
2183 return -ENOMEM;
2184 @@ -680,7 +1019,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
2185 return 0;
2186 }
2187
2188 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
2189 +struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
2190 if (node)
2191 node->refcount++;
2192
2193 @@ -703,7 +1042,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
2194 free(node);
2195 }
2196
2197 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
2198 +struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
2199 if (!node)
2200 return NULL;
2201
2202 @@ -714,7 +1053,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
2203 return NULL;
2204 }
2205
2206 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
2207 +struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
2208 if (index == 0)
2209 node = node->zero;
2210 else
2211 @@ -726,10 +1065,10 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
2212 return loc_network_tree_node_ref(node);
2213 }
2214
2215 -LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
2216 +int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
2217 return (!!node->network);
2218 }
2219
2220 -LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2221 +struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2222 return loc_network_ref(node->network);
2223 }
2224 diff --git a/src/perl/Location.xs b/src/perl/Location.xs
2225 index dcf3f0d..b7676d2 100644
2226 --- a/src/perl/Location.xs
2227 +++ b/src/perl/Location.xs
2228 @@ -125,7 +125,7 @@ database_countries(db)
2229 PPCODE:
2230 // Create Database enumerator
2231 struct loc_database_enumerator* enumerator;
2232 - int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
2233 + int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
2234
2235 if (err) {
2236 croak("Could not create a database enumerator\n");
2237 diff --git a/src/python/database.c b/src/python/database.c
2238 index 1013a58..f385c61 100644
2239 --- a/src/python/database.c
2240 +++ b/src/python/database.c
2241 @@ -17,6 +17,8 @@
2242 #include <Python.h>
2243
2244 #include <loc/libloc.h>
2245 +#include <loc/as.h>
2246 +#include <loc/as-list.h>
2247 #include <loc/database.h>
2248
2249 #include "locationmodule.h"
2250 @@ -207,10 +209,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
2251 return (PyObject*)self;
2252 }
2253
2254 -static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
2255 +static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
2256 struct loc_database_enumerator* enumerator;
2257
2258 - int r = loc_database_enumerator_new(&enumerator, self->db, what);
2259 + int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
2260 if (r) {
2261 PyErr_SetFromErrno(PyExc_SystemError);
2262 return NULL;
2263 @@ -223,7 +225,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
2264 }
2265
2266 static PyObject* Database_ases(DatabaseObject* self) {
2267 - return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
2268 + return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
2269 }
2270
2271 static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2272 @@ -234,7 +236,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2273
2274 struct loc_database_enumerator* enumerator;
2275
2276 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
2277 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
2278 if (r) {
2279 PyErr_SetFromErrno(PyExc_SystemError);
2280 return NULL;
2281 @@ -250,44 +252,142 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
2282 }
2283
2284 static PyObject* Database_networks(DatabaseObject* self) {
2285 - return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
2286 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
2287 +}
2288 +
2289 +static PyObject* Database_networks_flattened(DatabaseObject *self) {
2290 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
2291 }
2292
2293 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
2294 - char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
2295 - const char* country_code = NULL;
2296 - unsigned int asn = 0;
2297 + char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
2298 + PyObject* country_codes = NULL;
2299 + PyObject* asn_list = NULL;
2300 int flags = 0;
2301 int family = 0;
2302 + int flatten = 0;
2303
2304 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
2305 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
2306 + &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
2307 return NULL;
2308
2309 struct loc_database_enumerator* enumerator;
2310 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
2311 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
2312 + (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
2313 if (r) {
2314 PyErr_SetFromErrno(PyExc_SystemError);
2315 return NULL;
2316 }
2317
2318 // Set country code we are searching for
2319 - if (country_code) {
2320 - r = loc_database_enumerator_set_country_code(enumerator, country_code);
2321 + if (country_codes) {
2322 + struct loc_country_list* countries;
2323 + r = loc_country_list_new(loc_ctx, &countries);
2324 + if (r) {
2325 + PyErr_SetString(PyExc_SystemError, "Could not create country list");
2326 + return NULL;
2327 + }
2328 +
2329 + for (unsigned int i = 0; i < PyList_Size(country_codes); i++) {
2330 + PyObject* item = PyList_GetItem(country_codes, i);
2331 +
2332 + if (!PyUnicode_Check(item)) {
2333 + PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
2334 + loc_country_list_unref(countries);
2335 + return NULL;
2336 + }
2337 +
2338 + const char* country_code = PyUnicode_AsUTF8(item);
2339 +
2340 + struct loc_country* country;
2341 + r = loc_country_new(loc_ctx, &country, country_code);
2342 + if (r) {
2343 + if (r == -EINVAL) {
2344 + PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
2345 + } else {
2346 + PyErr_SetString(PyExc_SystemError, "Could not create country");
2347 + }
2348 +
2349 + loc_country_list_unref(countries);
2350 + return NULL;
2351 + }
2352 +
2353 + // Append it to the list
2354 + r = loc_country_list_append(countries, country);
2355 + if (r) {
2356 + PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
2357 +
2358 + loc_country_list_unref(countries);
2359 + loc_country_unref(country);
2360 + return NULL;
2361 + }
2362 +
2363 + loc_country_unref(country);
2364 + }
2365
2366 + r = loc_database_enumerator_set_countries(enumerator, countries);
2367 if (r) {
2368 PyErr_SetFromErrno(PyExc_SystemError);
2369 +
2370 + loc_country_list_unref(countries);
2371 return NULL;
2372 }
2373 +
2374 + loc_country_list_unref(countries);
2375 }
2376
2377 // Set the ASN we are searching for
2378 - if (asn) {
2379 - r = loc_database_enumerator_set_asn(enumerator, asn);
2380 + if (asn_list) {
2381 + struct loc_as_list* asns;
2382 + r = loc_as_list_new(loc_ctx, &asns);
2383 + if (r) {
2384 + PyErr_SetString(PyExc_SystemError, "Could not create AS list");
2385 + return NULL;
2386 + }
2387 +
2388 + for (unsigned int i = 0; i < PyList_Size(asn_list); i++) {
2389 + PyObject* item = PyList_GetItem(asn_list, i);
2390 +
2391 + if (!PyLong_Check(item)) {
2392 + PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
2393
2394 + loc_as_list_unref(asns);
2395 + return NULL;
2396 + }
2397 +
2398 + unsigned long number = PyLong_AsLong(item);
2399 +
2400 + struct loc_as* as;
2401 + r = loc_as_new(loc_ctx, &as, number);
2402 + if (r) {
2403 + PyErr_SetString(PyExc_SystemError, "Could not create AS");
2404 +
2405 + loc_as_list_unref(asns);
2406 + loc_as_unref(as);
2407 + return NULL;
2408 + }
2409 +
2410 + r = loc_as_list_append(asns, as);
2411 + if (r) {
2412 + PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
2413 +
2414 + loc_as_list_unref(asns);
2415 + loc_as_unref(as);
2416 + return NULL;
2417 + }
2418 +
2419 + loc_as_unref(as);
2420 + }
2421 +
2422 + r = loc_database_enumerator_set_asns(enumerator, asns);
2423 if (r) {
2424 PyErr_SetFromErrno(PyExc_SystemError);
2425 +
2426 + loc_as_list_unref(asns);
2427 return NULL;
2428 }
2429 +
2430 + loc_as_list_unref(asns);
2431 }
2432
2433 // Set the flags we are searching for
2434 @@ -317,7 +417,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
2435 }
2436
2437 static PyObject* Database_countries(DatabaseObject* self) {
2438 - return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
2439 + return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
2440 }
2441
2442 static struct PyMethodDef Database_methods[] = {
2443 @@ -403,6 +503,13 @@ static struct PyGetSetDef Database_getsetters[] = {
2444 NULL,
2445 NULL,
2446 },
2447 + {
2448 + "networks_flattened",
2449 + (getter)Database_networks_flattened,
2450 + NULL,
2451 + NULL,
2452 + NULL,
2453 + },
2454 {
2455 "vendor",
2456 (getter)Database_get_vendor,
2457 diff --git a/src/python/downloader.py b/src/python/downloader.py
2458 index 87bbb68..05f7872 100644
2459 --- a/src/python/downloader.py
2460 +++ b/src/python/downloader.py
2461 @@ -119,8 +119,8 @@ class Downloader(object):
2462
2463 headers = {}
2464 if timestamp:
2465 - headers["If-Modified-Since"] = timestamp.strftime(
2466 - "%a, %d %b %Y %H:%M:%S GMT",
2467 + headers["If-Modified-Since"] = time.strftime(
2468 + "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
2469 )
2470
2471 t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
2472 @@ -195,7 +195,7 @@ class Downloader(object):
2473 db = Database(f.name)
2474
2475 # Database is not recent
2476 - if timestamp and db.created_at < timestamp.timestamp():
2477 + if timestamp and db.created_at < timestamp:
2478 return False
2479
2480 log.info("Downloaded new database from %s" % (time.strftime(
2481 diff --git a/src/python/export.py b/src/python/export.py
2482 index d15c6f0..f0eae26 100644
2483 --- a/src/python/export.py
2484 +++ b/src/python/export.py
2485 @@ -29,7 +29,7 @@ import _location
2486 log = logging.getLogger("location.export")
2487 log.propagate = 1
2488
2489 -flags = {
2490 +FLAGS = {
2491 _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
2492 _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
2493 _location.NETWORK_FLAG_ANYCAST : "A3",
2494 @@ -39,11 +39,8 @@ class OutputWriter(object):
2495 suffix = "networks"
2496 mode = "w"
2497
2498 - def __init__(self, f, prefix=None, flatten=True):
2499 - self.f, self.prefix, self.flatten = f, prefix, flatten
2500 -
2501 - # The previously written network
2502 - self._last_network = None
2503 + def __init__(self, f, prefix=None):
2504 + self.f, self.prefix = f, prefix
2505
2506 # Immediately write the header
2507 self._write_header()
2508 @@ -60,18 +57,6 @@ class OutputWriter(object):
2509 def __repr__(self):
2510 return "<%s f=%s>" % (self.__class__.__name__, self.f)
2511
2512 - def _flatten(self, network):
2513 - """
2514 - Checks if the given network needs to be written to file,
2515 - or if it is a subnet of the previously written network.
2516 - """
2517 - if self._last_network and network.is_subnet_of(self._last_network):
2518 - return True
2519 -
2520 - # Remember this network for the next call
2521 - self._last_network = network
2522 - return False
2523 -
2524 def _write_header(self):
2525 """
2526 The header of the file
2527 @@ -84,16 +69,8 @@ class OutputWriter(object):
2528 """
2529 pass
2530
2531 - def _write_network(self, network):
2532 - self.f.write("%s\n" % network)
2533 -
2534 def write(self, network):
2535 - if self.flatten and self._flatten(network):
2536 - log.debug("Skipping writing network %s" % network)
2537 - return
2538 -
2539 - # Write the network to file
2540 - self._write_network(network)
2541 + self.f.write("%s\n" % network)
2542
2543 def finish(self):
2544 """
2545 @@ -114,7 +91,7 @@ class IpsetOutputWriter(OutputWriter):
2546 def _write_header(self):
2547 self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
2548
2549 - def _write_network(self, network):
2550 + def write(self, network):
2551 self.f.write("add %s %s\n" % (self.prefix, network))
2552
2553
2554 @@ -130,7 +107,7 @@ class NftablesOutputWriter(OutputWriter):
2555 def _write_footer(self):
2556 self.f.write("}\n")
2557
2558 - def _write_network(self, network):
2559 + def write(self, network):
2560 self.f.write(" %s,\n" % network)
2561
2562
2563 @@ -142,14 +119,9 @@ class XTGeoIPOutputWriter(OutputWriter):
2564 suffix = "iv"
2565 mode = "wb"
2566
2567 - def _write_network(self, network):
2568 - for address in (network.first_address, network.last_address):
2569 - # Convert this into a string of bits
2570 - bytes = socket.inet_pton(
2571 - network.family, address,
2572 - )
2573 -
2574 - self.f.write(bytes)
2575 + def write(self, network):
2576 + self.f.write(network._first_address)
2577 + self.f.write(network._last_address)
2578
2579
2580 formats = {
2581 @@ -185,8 +157,14 @@ class Exporter(object):
2582
2583 writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
2584
2585 + # Filter countries from special country codes
2586 + country_codes = [
2587 + country_code for country_code in countries if not country_code in FLAGS.values()
2588 + ]
2589 +
2590 # Get all networks that match the family
2591 - networks = self.db.search_networks(family=family)
2592 + networks = self.db.search_networks(family=family,
2593 + country_codes=country_codes, asns=asns, flatten=True)
2594
2595 # Walk through all networks
2596 for network in networks:
2597 @@ -203,10 +181,10 @@ class Exporter(object):
2598 pass
2599
2600 # Handle flags
2601 - for flag in flags:
2602 + for flag in FLAGS:
2603 if network.has_flag(flag):
2604 # Fetch the "fake" country code
2605 - country = flags[flag]
2606 + country = FLAGS[flag]
2607
2608 try:
2609 writers[country].write(network)
2610 diff --git a/src/python/importer.py b/src/python/importer.py
2611 index f19db4b..5f46bc3 100644
2612 --- a/src/python/importer.py
2613 +++ b/src/python/importer.py
2614 @@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
2615 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
2616
2617 # Latin America and Caribbean Network Information Centre
2618 - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2619 + "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2620
2621 # Réseaux IP Européens
2622 #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
2623 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2624 index 1467923..2dec89e 100644
2625 --- a/src/python/location-importer.in
2626 +++ b/src/python/location-importer.in
2627 @@ -152,6 +152,7 @@ class CLI(object):
2628 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
2629 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
2630 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
2631 + CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
2632
2633 -- autnums
2634 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
2635 @@ -165,6 +166,7 @@ class CLI(object):
2636 -- networks
2637 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
2638 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
2639 + CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
2640 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
2641
2642 -- overrides
2643 @@ -188,6 +190,8 @@ class CLI(object):
2644 );
2645 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
2646 ON network_overrides(network);
2647 + CREATE INDEX IF NOT EXISTS network_overrides_search
2648 + ON network_overrides USING GIST(network inet_ops);
2649 """)
2650
2651 return db
2652 @@ -234,32 +238,24 @@ class CLI(object):
2653
2654 # Select all known networks
2655 rows = self.db.query("""
2656 - -- Get a (sorted) list of all known networks
2657 - WITH known_networks AS (
2658 - SELECT network FROM announcements
2659 - UNION
2660 - SELECT network FROM networks
2661 - ORDER BY network
2662 - )
2663 -
2664 -- Return a list of those networks enriched with all
2665 -- other information that we store in the database
2666 SELECT
2667 - DISTINCT ON (known_networks.network)
2668 - known_networks.network AS network,
2669 - announcements.autnum AS autnum,
2670 + DISTINCT ON (network)
2671 + network,
2672 + autnum,
2673
2674 -- Country
2675 COALESCE(
2676 (
2677 SELECT country FROM network_overrides overrides
2678 - WHERE announcements.network <<= overrides.network
2679 + WHERE networks.network <<= overrides.network
2680 ORDER BY masklen(overrides.network) DESC
2681 LIMIT 1
2682 ),
2683 (
2684 SELECT country FROM autnum_overrides overrides
2685 - WHERE announcements.autnum = overrides.number
2686 + WHERE networks.autnum = overrides.number
2687 ),
2688 networks.country
2689 ) AS country,
2690 @@ -268,50 +264,67 @@ class CLI(object):
2691 COALESCE(
2692 (
2693 SELECT is_anonymous_proxy FROM network_overrides overrides
2694 - WHERE announcements.network <<= overrides.network
2695 + WHERE networks.network <<= overrides.network
2696 ORDER BY masklen(overrides.network) DESC
2697 LIMIT 1
2698 ),
2699 (
2700 SELECT is_anonymous_proxy FROM autnum_overrides overrides
2701 - WHERE announcements.autnum = overrides.number
2702 + WHERE networks.autnum = overrides.number
2703 ),
2704 FALSE
2705 ) AS is_anonymous_proxy,
2706 COALESCE(
2707 (
2708 SELECT is_satellite_provider FROM network_overrides overrides
2709 - WHERE announcements.network <<= overrides.network
2710 + WHERE networks.network <<= overrides.network
2711 ORDER BY masklen(overrides.network) DESC
2712 LIMIT 1
2713 ),
2714 (
2715 SELECT is_satellite_provider FROM autnum_overrides overrides
2716 - WHERE announcements.autnum = overrides.number
2717 + WHERE networks.autnum = overrides.number
2718 ),
2719 FALSE
2720 ) AS is_satellite_provider,
2721 COALESCE(
2722 (
2723 SELECT is_anycast FROM network_overrides overrides
2724 - WHERE announcements.network <<= overrides.network
2725 + WHERE networks.network <<= overrides.network
2726 ORDER BY masklen(overrides.network) DESC
2727 LIMIT 1
2728 ),
2729 (
2730 SELECT is_anycast FROM autnum_overrides overrides
2731 - WHERE announcements.autnum = overrides.number
2732 + WHERE networks.autnum = overrides.number
2733 ),
2734 FALSE
2735 - ) AS is_anycast,
2736 -
2737 - -- Must be part of returned values for ORDER BY clause
2738 - masklen(announcements.network) AS sort_a,
2739 - masklen(networks.network) AS sort_b
2740 - FROM known_networks
2741 - LEFT JOIN announcements ON known_networks.network <<= announcements.network
2742 - LEFT JOIN networks ON known_networks.network <<= networks.network
2743 - ORDER BY known_networks.network, sort_a DESC, sort_b DESC
2744 + ) AS is_anycast
2745 + FROM (
2746 + SELECT
2747 + known_networks.network AS network,
2748 + announcements.autnum AS autnum,
2749 + networks.country AS country,
2750 +
2751 + -- Must be part of returned values for ORDER BY clause
2752 + masklen(announcements.network) AS sort_a,
2753 + masklen(networks.network) AS sort_b
2754 + FROM (
2755 + SELECT network FROM announcements
2756 + UNION ALL
2757 + SELECT network FROM networks
2758 + UNION ALL
2759 + SELECT network FROM network_overrides
2760 + ) known_networks
2761 + LEFT JOIN
2762 + announcements ON known_networks.network <<= announcements.network
2763 + LEFT JOIN
2764 + networks ON known_networks.network <<= networks.network
2765 + ORDER BY
2766 + known_networks.network,
2767 + sort_a DESC,
2768 + sort_b DESC
2769 + ) networks
2770 """)
2771
2772 for row in rows:
2773 @@ -363,6 +376,16 @@ class CLI(object):
2774 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
2775 ON COMMIT DROP;
2776 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
2777 +
2778 + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
2779 + ON COMMIT DROP;
2780 + CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
2781 + CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
2782 + """)
2783 +
2784 + # Remove all previously imported content
2785 + self.db.execute("""
2786 + TRUNCATE TABLE networks;
2787 """)
2788
2789 for source in location.importer.WHOIS_SOURCES:
2790 @@ -370,31 +393,72 @@ class CLI(object):
2791 for block in f:
2792 self._parse_block(block)
2793
2794 + # Process all parsed networks from every RIR we happen to have access to,
2795 + # insert the largest network chunks into the networks table immediately...
2796 + families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
2797 +
2798 + for family in (row.family for row in families):
2799 + smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
2800 +
2801 + self.db.execute("INSERT INTO networks(network, country) \
2802 + SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
2803 +
2804 + # ... determine any other prefixes for this network family, ...
2805 + prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
2806 + WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
2807 +
2808 + # ... and insert networks with this prefix in case they provide additional
2809 + # information (i. e. subnet of a larger chunk with a different country)
2810 + for prefix in (row.prefix for row in prefixes):
2811 + self.db.execute("""
2812 + WITH candidates AS (
2813 + SELECT
2814 + _rirdata.network,
2815 + _rirdata.country
2816 + FROM
2817 + _rirdata
2818 + WHERE
2819 + family(_rirdata.network) = %s
2820 + AND
2821 + masklen(_rirdata.network) = %s
2822 + ),
2823 + filtered AS (
2824 + SELECT
2825 + DISTINCT ON (c.network)
2826 + c.network,
2827 + c.country,
2828 + masklen(networks.network),
2829 + networks.country AS parent_country
2830 + FROM
2831 + candidates c
2832 + LEFT JOIN
2833 + networks
2834 + ON
2835 + c.network << networks.network
2836 + ORDER BY
2837 + c.network,
2838 + masklen(networks.network) DESC NULLS LAST
2839 + )
2840 + INSERT INTO
2841 + networks(network, country)
2842 + SELECT
2843 + network,
2844 + country
2845 + FROM
2846 + filtered
2847 + WHERE
2848 + parent_country IS NULL
2849 + OR
2850 + country <> parent_country
2851 + ON CONFLICT DO NOTHING""",
2852 + family, prefix,
2853 + )
2854 +
2855 self.db.execute("""
2856 INSERT INTO autnums(number, name)
2857 SELECT _autnums.number, _organizations.name FROM _autnums
2858 JOIN _organizations ON _autnums.organization = _organizations.handle
2859 - ON CONFLICT (number) DO UPDATE SET name = excluded.name
2860 - """)
2861 -
2862 - self.db.execute("""
2863 - --- Purge any redundant entries
2864 - CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
2865 - AS
2866 - SELECT network FROM networks candidates
2867 - WHERE EXISTS (
2868 - SELECT FROM networks
2869 - WHERE
2870 - networks.network << candidates.network
2871 - AND
2872 - networks.country = candidates.country
2873 - );
2874 -
2875 - CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
2876 -
2877 - DELETE FROM networks WHERE EXISTS (
2878 - SELECT FROM _garbage WHERE networks.network = _garbage.network
2879 - );
2880 + ON CONFLICT (number) DO UPDATE SET name = excluded.name;
2881 """)
2882
2883 # Download all extended sources
2884 @@ -405,6 +469,69 @@ class CLI(object):
2885 for line in f:
2886 self._parse_line(line)
2887
2888 + def _check_parsed_network(self, network):
2889 + """
2890 + Assistive function to detect and subsequently sort out parsed
2891 + networks from RIR data (both Whois and so-called "extended sources"),
2892 + which are or have...
2893 +
2894 + (a) not globally routable (RFC 1918 space, et al.)
2895 + (b) covering a too large chunk of the IP address space (prefix length
2896 + is < 7 for IPv4 networks, and < 10 for IPv6)
2897 + (c) "0.0.0.0" or "::" as a network address
2898 + (d) are too small for being publicly announced (we have decided not to
2899 + process them at the moment, as they significantly enlarge our
2900 + database without providing very helpful additional information)
2901 +
2902 + This unfortunately is necessary due to brain-dead clutter across
2903 + various RIR databases, causing mismatches and eventually disruptions.
2904 +
2905 + We will return False in case a network is not suitable for adding
2906 + it to our database, and True otherwise.
2907 + """
2908 +
2909 + if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
2910 + return False
2911 +
2912 + if not network.is_global:
2913 + log.warning("Skipping non-globally routable network: %s" % network)
2914 + return False
2915 +
2916 + if network.version == 4:
2917 + if network.prefixlen < 7:
2918 + log.warning("Skipping too big IP chunk: %s" % network)
2919 + return False
2920 +
2921 + if network.prefixlen > 24:
2922 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2923 + return False
2924 +
2925 + if str(network.network_address) == "0.0.0.0":
2926 + log.warning("Skipping network based on 0.0.0.0: %s" % network)
2927 + return False
2928 +
2929 + elif network.version == 6:
2930 + if network.prefixlen < 10:
2931 + log.warning("Skipping too big IP chunk: %s" % network)
2932 + return False
2933 +
2934 + if network.prefixlen > 48:
2935 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2936 + return False
2937 +
2938 + if str(network.network_address) == "::":
2939 + log.warning("Skipping network based on '::': %s" % network)
2940 + return False
2941 +
2942 + else:
2943 + # This should not happen...
2944 + log.warning("Skipping network of unknown family, this should not happen: %s" % network)
2945 + return False
2946 +
2947 + # In case we have made it here, the network is considered to
2948 + # be suitable for libloc consumption...
2949 + return True
2950 +
2951 def _parse_block(self, block):
2952 # Get first line to find out what type of block this is
2953 line = block[0]
2954 @@ -433,7 +560,7 @@ class CLI(object):
2955 autnum["asn"] = m.group(2)
2956
2957 elif key == "org":
2958 - autnum[key] = val
2959 + autnum[key] = val.upper()
2960
2961 # Skip empty objects
2962 if not autnum:
2963 @@ -447,15 +574,22 @@ class CLI(object):
2964 )
2965
2966 def _parse_inetnum_block(self, block):
2967 - logging.debug("Parsing inetnum block:")
2968 + log.debug("Parsing inetnum block:")
2969
2970 inetnum = {}
2971 for line in block:
2972 - logging.debug(line)
2973 + log.debug(line)
2974
2975 # Split line
2976 key, val = split_line(line)
2977
2978 + # Filter any inetnum records which are only referring to IP space
2979 + # not managed by that specific RIR...
2980 + if key == "netname":
2981 + if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
2982 + log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
2983 + return
2984 +
2985 if key == "inetnum":
2986 start_address, delim, end_address = val.partition("-")
2987
2988 @@ -467,7 +601,7 @@ class CLI(object):
2989 start_address = ipaddress.ip_address(start_address)
2990 end_address = ipaddress.ip_address(end_address)
2991 except ValueError:
2992 - logging.warning("Could not parse line: %s" % line)
2993 + log.warning("Could not parse line: %s" % line)
2994 return
2995
2996 # Set prefix to default
2997 @@ -484,23 +618,24 @@ class CLI(object):
2998 inetnum[key] = val
2999
3000 elif key == "country":
3001 - if val == "UNITED STATES":
3002 - val = "US"
3003 -
3004 inetnum[key] = val.upper()
3005
3006 # Skip empty objects
3007 - if not inetnum:
3008 + if not inetnum or not "country" in inetnum:
3009 + return
3010 +
3011 + # Skip objects with bogus country code 'ZZ'
3012 + if inetnum.get("country") == "ZZ":
3013 + log.warning("Skipping network with bogus country 'ZZ': %s" % \
3014 + (inetnum.get("inet6num") or inetnum.get("inetnum")))
3015 return
3016
3017 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
3018
3019 - # Bail out in case we have processed a non-public IP network
3020 - if network.is_private:
3021 - logging.warning("Skipping non-globally routable network: %s" % network)
3022 + if not self._check_parsed_network(network):
3023 return
3024
3025 - self.db.execute("INSERT INTO networks(network, country) \
3026 + self.db.execute("INSERT INTO _rirdata(network, country) \
3027 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
3028 "%s" % network, inetnum.get("country"),
3029 )
3030 @@ -511,7 +646,9 @@ class CLI(object):
3031 # Split line
3032 key, val = split_line(line)
3033
3034 - if key in ("organisation", "org-name"):
3035 + if key == "organisation":
3036 + org[key] = val.upper()
3037 + elif key == "org-name":
3038 org[key] = val
3039
3040 # Skip empty objects
3041 @@ -581,6 +718,9 @@ class CLI(object):
3042 log.warning("Invalid IP address: %s" % address)
3043 return
3044
3045 + if not self._check_parsed_network(network):
3046 + return
3047 +
3048 self.db.execute("INSERT INTO networks(network, country) \
3049 VALUES(%s, %s) ON CONFLICT (network) DO \
3050 UPDATE SET country = excluded.country",
3051 diff --git a/src/python/location.in b/src/python/location.in
3052 index 44ad726..b30beae 100644
3053 --- a/src/python/location.in
3054 +++ b/src/python/location.in
3055 @@ -253,6 +253,7 @@ class CLI(object):
3056 network = db.lookup(address)
3057 except ValueError:
3058 print(_("Invalid IP address: %s") % address, file=sys.stderr)
3059 + return 2
3060
3061 args = {
3062 "address" : address,
3063 @@ -398,10 +399,7 @@ class CLI(object):
3064
3065 def handle_update(self, db, ns):
3066 if ns.cron and db:
3067 - now = datetime.datetime.utcnow()
3068 -
3069 - # Parse the database timestamp
3070 - t = datetime.datetime.utcfromtimestamp(db.created_at)
3071 + now = time.time()
3072
3073 if ns.cron == "daily":
3074 delta = datetime.timedelta(days=1)
3075 @@ -410,22 +408,20 @@ class CLI(object):
3076 elif ns.cron == "monthly":
3077 delta = datetime.timedelta(days=30)
3078
3079 + delta = delta.total_seconds()
3080 +
3081 # Check if the database has recently been updated
3082 - if t >= (now - delta):
3083 + if db.created_at >= (now - delta):
3084 log.info(
3085 - _("The database has been updated recently (%s)") % \
3086 - format_timedelta(now - t),
3087 + _("The database has been updated recently"),
3088 )
3089 return 3
3090
3091 # Fetch the timestamp we need from DNS
3092 t = location.discover_latest_version()
3093
3094 - # Parse timestamp into datetime format
3095 - timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
3096 -
3097 # Check the version of the local database
3098 - if db and timestamp and db.created_at >= timestamp.timestamp():
3099 + if db and t and db.created_at >= t:
3100 log.info("Already on the latest version")
3101 return
3102
3103 @@ -437,7 +433,7 @@ class CLI(object):
3104
3105 # Try downloading a new database
3106 try:
3107 - t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
3108 + t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
3109
3110 # If no file could be downloaded, log a message
3111 except FileNotFoundError as e:
3112 @@ -453,13 +449,7 @@ class CLI(object):
3113
3114 return 0
3115
3116 - def handle_verify(self, ns):
3117 - try:
3118 - db = location.Database(ns.database)
3119 - except FileNotFoundError as e:
3120 - log.error("%s: %s" % (ns.database, e))
3121 - return 127
3122 -
3123 + def handle_verify(self, db, ns):
3124 # Verify the database
3125 with open(ns.public_key, "r") as f:
3126 if not db.verify(f):
3127 diff --git a/src/python/network.c b/src/python/network.c
3128 index 5496d1e..b6e92fb 100644
3129 --- a/src/python/network.c
3130 +++ b/src/python/network.c
3131 @@ -20,10 +20,29 @@
3132
3133 #include <loc/libloc.h>
3134 #include <loc/network.h>
3135 +#include <loc/network-list.h>
3136
3137 #include "locationmodule.h"
3138 #include "network.h"
3139
3140 +static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
3141 + PyObject* list = PyList_New(0);
3142 + if (!networks)
3143 + return list;
3144 +
3145 + while (!loc_network_list_empty(networks)) {
3146 + struct loc_network* network = loc_network_list_pop(networks);
3147 +
3148 + PyObject* n = new_network(&NetworkType, network);
3149 + PyList_Append(list, n);
3150 +
3151 + loc_network_unref(network);
3152 + Py_DECREF(n);
3153 + }
3154 +
3155 + return list;
3156 +}
3157 +
3158 PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
3159 NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
3160 if (self) {
3161 @@ -154,13 +173,28 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
3162 Py_RETURN_NONE;
3163 }
3164
3165 +static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
3166 + NetworkObject* other = NULL;
3167 +
3168 + if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3169 + return NULL;
3170 +
3171 + struct loc_network_list* list = loc_network_exclude(self->network, other->network);
3172 +
3173 + // Convert to Python objects
3174 + PyObject* obj = PyList_FromNetworkList(list);
3175 + loc_network_list_unref(list);
3176 +
3177 + return obj;
3178 +}
3179 +
3180 static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
3181 NetworkObject* other = NULL;
3182
3183 if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3184 return NULL;
3185
3186 - if (loc_network_is_subnet_of(self->network, other->network))
3187 + if (loc_network_is_subnet(other->network, self->network))
3188 Py_RETURN_TRUE;
3189
3190 Py_RETURN_FALSE;
3191 @@ -181,6 +215,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) {
3192 return obj;
3193 }
3194
3195 +static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) {
3196 + struct in_addr address4;
3197 +
3198 + // Convert IPv4 addresses to struct in_addr
3199 + if (IN6_IS_ADDR_V4MAPPED(address6)) {
3200 + address4.s_addr = address6->s6_addr32[3];
3201 +
3202 + return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4));
3203 + }
3204 +
3205 + // Return IPv6 addresses as they are
3206 + return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6));
3207 +}
3208 +
3209 +static PyObject* Network_get__first_address(NetworkObject* self) {
3210 + const struct in6_addr* address = loc_network_get_first_address(self->network);
3211 +
3212 + return PyBytes_FromAddress(address);
3213 +}
3214 +
3215 static PyObject* Network_get_last_address(NetworkObject* self) {
3216 char* address = loc_network_format_last_address(self->network);
3217
3218 @@ -190,7 +244,19 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
3219 return obj;
3220 }
3221
3222 +static PyObject* Network_get__last_address(NetworkObject* self) {
3223 + const struct in6_addr* address = loc_network_get_last_address(self->network);
3224 +
3225 + return PyBytes_FromAddress(address);
3226 +}
3227 +
3228 static struct PyMethodDef Network_methods[] = {
3229 + {
3230 + "exclude",
3231 + (PyCFunction)Network_exclude,
3232 + METH_VARARGS,
3233 + NULL,
3234 + },
3235 {
3236 "has_flag",
3237 (PyCFunction)Network_has_flag,
3238 @@ -241,6 +307,13 @@ static struct PyGetSetDef Network_getsetters[] = {
3239 NULL,
3240 NULL,
3241 },
3242 + {
3243 + "_first_address",
3244 + (getter)Network_get__first_address,
3245 + NULL,
3246 + NULL,
3247 + NULL,
3248 + },
3249 {
3250 "last_address",
3251 (getter)Network_get_last_address,
3252 @@ -248,6 +321,13 @@ static struct PyGetSetDef Network_getsetters[] = {
3253 NULL,
3254 NULL,
3255 },
3256 + {
3257 + "_last_address",
3258 + (getter)Network_get__last_address,
3259 + NULL,
3260 + NULL,
3261 + NULL,
3262 + },
3263 { NULL },
3264 };
3265
3266 diff --git a/src/test-as.c b/src/test-as.c
3267 index 839a04c..2d61675 100644
3268 --- a/src/test-as.c
3269 +++ b/src/test-as.c
3270 @@ -95,7 +95,7 @@ int main(int argc, char** argv) {
3271 // Enumerator
3272
3273 struct loc_database_enumerator* enumerator;
3274 - err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
3275 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
3276 if (err) {
3277 fprintf(stderr, "Could not create a database enumerator\n");
3278 exit(EXIT_FAILURE);
3279 diff --git a/src/test-database.c b/src/test-database.c
3280 index b4a75c4..da4b11c 100644
3281 --- a/src/test-database.c
3282 +++ b/src/test-database.c
3283 @@ -38,6 +38,14 @@ const char* DESCRIPTION =
3284 "Maecenas ut venenatis nunc.";
3285 const char* LICENSE = "CC";
3286
3287 +const char* networks[] = {
3288 + "2001:db8::/32",
3289 + "2001:db8:1000::/48",
3290 + "2001:db8:2000::/48",
3291 + "2001:db8:2020::/48",
3292 + NULL,
3293 +};
3294 +
3295 static int attempt_to_open(struct loc_ctx* ctx, char* path) {
3296 FILE* f = fopen(path, "r");
3297 if (!f)
3298 @@ -139,6 +147,24 @@ int main(int argc, char** argv) {
3299 exit(EXIT_FAILURE);
3300 }
3301
3302 + struct loc_network* network = NULL;
3303 +
3304 + // Add some networks
3305 + const char** n = networks;
3306 + while (*n) {
3307 + err = loc_writer_add_network(writer, &network, *n);
3308 + if (err) {
3309 + fprintf(stderr, "Could not add network %s\n", *n);
3310 + exit(EXIT_FAILURE);
3311 + }
3312 +
3313 + // Set a country
3314 + loc_network_set_country_code(network, "XX");
3315 +
3316 + // Next one
3317 + n++;
3318 + }
3319 +
3320 FILE* f = tmpfile();
3321 if (!f) {
3322 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
3323 @@ -170,6 +196,33 @@ int main(int argc, char** argv) {
3324 exit(EXIT_FAILURE);
3325 }
3326
3327 + // Enumerator
3328 + struct loc_database_enumerator* enumerator;
3329 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
3330 + if (err) {
3331 + fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
3332 + exit(EXIT_FAILURE);
3333 + }
3334 +
3335 + // Walk through all networks
3336 + while (1) {
3337 + err = loc_database_enumerator_next_network(enumerator, &network);
3338 + if (err) {
3339 + fprintf(stderr, "Error fetching the next network: %d\n", err);
3340 + exit(EXIT_FAILURE);
3341 + }
3342 +
3343 + if (!network)
3344 + break;
3345 +
3346 + char* s = loc_network_str(network);
3347 + printf("Got network: %s\n", s);
3348 + free(s);
3349 + }
3350 +
3351 + // Free the enumerator
3352 + loc_database_enumerator_unref(enumerator);
3353 +
3354 // Close the database
3355 loc_database_unref(db);
3356 loc_unref(ctx);
3357 diff --git a/src/test-network-list.c b/src/test-network-list.c
3358 new file mode 100644
3359 index 0000000..6f32ff7
3360 --- /dev/null
3361 +++ b/src/test-network-list.c
3362 @@ -0,0 +1,183 @@
3363 +/*
3364 + libloc - A library to determine the location of someone on the Internet
3365 +
3366 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
3367 +
3368 + This program is free software; you can redistribute it and/or modify
3369 + it under the terms of the GNU General Public License as published by
3370 + the Free Software Foundation; either version 2 of the License, or
3371 + (at your option) any later version.
3372 +
3373 + This program is distributed in the hope that it will be useful,
3374 + but WITHOUT ANY WARRANTY; without even the implied warranty of
3375 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3376 + GNU General Public License for more details.
3377 +*/
3378 +
3379 +#include <errno.h>
3380 +#include <stdio.h>
3381 +#include <stddef.h>
3382 +#include <stdlib.h>
3383 +#include <string.h>
3384 +#include <syslog.h>
3385 +
3386 +#include <loc/libloc.h>
3387 +#include <loc/network.h>
3388 +#include <loc/network-list.h>
3389 +
3390 +int main(int argc, char** argv) {
3391 + int err;
3392 +
3393 + struct loc_ctx* ctx;
3394 + err = loc_new(&ctx);
3395 + if (err < 0)
3396 + exit(EXIT_FAILURE);
3397 +
3398 + // Enable debug logging
3399 + loc_set_log_priority(ctx, LOG_DEBUG);
3400 +
3401 + // Create a network
3402 + struct loc_network* network1;
3403 + err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32");
3404 + if (err) {
3405 + fprintf(stderr, "Could not create the network1\n");
3406 + exit(EXIT_FAILURE);
3407 + }
3408 +
3409 + struct loc_network* subnet1;
3410 + err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48");
3411 + if (err) {
3412 + fprintf(stderr, "Could not create the subnet1\n");
3413 + exit(EXIT_FAILURE);
3414 + }
3415 +
3416 + struct loc_network* subnet2;
3417 + err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48");
3418 + if (err) {
3419 + fprintf(stderr, "Could not create the subnet2\n");
3420 + exit(EXIT_FAILURE);
3421 + }
3422 +
3423 + struct loc_network* subnet3;
3424 + err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48");
3425 + if (err) {
3426 + fprintf(stderr, "Could not create the subnet3\n");
3427 + exit(EXIT_FAILURE);
3428 + }
3429 +
3430 + struct loc_network* subnet4;
3431 + err = loc_network_new_from_string(ctx, &subnet4, "2001:db8:d::/48");
3432 + if (err) {
3433 + fprintf(stderr, "Could not create the subnet4\n");
3434 + exit(EXIT_FAILURE);
3435 + }
3436 +
3437 + struct loc_network* subnet5;
3438 + err = loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48");
3439 + if (err) {
3440 + fprintf(stderr, "Could not create the subnet5\n");
3441 + exit(EXIT_FAILURE);
3442 + }
3443 +
3444 + struct loc_network* subnet6;
3445 + err = loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48");
3446 + if (err) {
3447 + fprintf(stderr, "Could not create the subnet6\n");
3448 + exit(EXIT_FAILURE);
3449 + }
3450 +
3451 + // Make a list with both subnets
3452 + struct loc_network_list* subnets;
3453 + err = loc_network_list_new(ctx, &subnets);
3454 + if (err) {
3455 + fprintf(stderr, "Could not create subnets list\n");
3456 + exit(EXIT_FAILURE);
3457 + }
3458 +
3459 + size_t size = loc_network_list_size(subnets);
3460 + if (size > 0) {
3461 + fprintf(stderr, "The list is not empty: %zu\n", size);
3462 + exit(EXIT_FAILURE);
3463 + }
3464 +
3465 + err = loc_network_list_push(subnets, subnet1);
3466 + if (err) {
3467 + fprintf(stderr, "Could not add subnet1 to subnets list\n");
3468 + exit(EXIT_FAILURE);
3469 + }
3470 +
3471 + if (loc_network_list_empty(subnets)) {
3472 + fprintf(stderr, "The subnets list reports that it is empty\n");
3473 + exit(EXIT_FAILURE);
3474 + }
3475 +
3476 + err = loc_network_list_push(subnets, subnet2);
3477 + if (err) {
3478 + fprintf(stderr, "Could not add subnet2 to subnets list\n");
3479 + exit(EXIT_FAILURE);
3480 + }
3481 +
3482 + // Add the fourth one next
3483 + err = loc_network_list_push(subnets, subnet4);
3484 + if (err) {
3485 + fprintf(stderr, "Could not add subnet4 to subnets list\n");
3486 + exit(EXIT_FAILURE);
3487 + }
3488 +
3489 + // Add the third one
3490 + err = loc_network_list_push(subnets, subnet3);
3491 + if (err) {
3492 + fprintf(stderr, "Could not add subnet3 to subnets list\n");
3493 + exit(EXIT_FAILURE);
3494 + }
3495 +
3496 + // Add more subnets
3497 + err = loc_network_list_push(subnets, subnet5);
3498 + if (err) {
3499 + fprintf(stderr, "Could not add subnet5 to subnets list\n");
3500 + exit(EXIT_FAILURE);
3501 + }
3502 +
3503 + err = loc_network_list_push(subnets, subnet6);
3504 + if (err) {
3505 + fprintf(stderr, "Could not add subnet6 to subnets list\n");
3506 + exit(EXIT_FAILURE);
3507 + }
3508 +
3509 + loc_network_list_dump(subnets);
3510 +
3511 + size = loc_network_list_size(subnets);
3512 + if (size != 6) {
3513 + fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
3514 + exit(EXIT_FAILURE);
3515 + }
3516 +
3517 + // Exclude subnet1 from network1
3518 + struct loc_network_list* excluded = loc_network_exclude(network1, subnet1);
3519 + if (!excluded) {
3520 + fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n");
3521 + exit(EXIT_FAILURE);
3522 + }
3523 +
3524 + loc_network_list_dump(excluded);
3525 +
3526 + // Exclude all subnets from network1
3527 + excluded = loc_network_exclude_list(network1, subnets);
3528 + if (!excluded) {
3529 + fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n");
3530 + exit(EXIT_FAILURE);
3531 + }
3532 +
3533 + loc_network_list_dump(excluded);
3534 +
3535 + if (excluded)
3536 + loc_network_list_unref(excluded);
3537 +
3538 + loc_network_list_unref(subnets);
3539 + loc_network_unref(network1);
3540 + loc_network_unref(subnet1);
3541 + loc_network_unref(subnet2);
3542 + loc_unref(ctx);
3543 +
3544 + return EXIT_SUCCESS;
3545 +}
3546 diff --git a/src/test-network.c b/src/test-network.c
3547 index d38f13d..dde13f1 100644
3548 --- a/src/test-network.c
3549 +++ b/src/test-network.c
3550 @@ -14,6 +14,7 @@
3551 GNU General Public License for more details.
3552 */
3553
3554 +#include <arpa/inet.h>
3555 #include <errno.h>
3556 #include <stdio.h>
3557 #include <stddef.h>
3558 @@ -37,12 +38,21 @@ int main(int argc, char** argv) {
3559 // Enable debug logging
3560 loc_set_log_priority(ctx, LOG_DEBUG);
3561
3562 +#if 0
3563 struct loc_network_tree* tree;
3564 err = loc_network_tree_new(ctx, &tree);
3565 if (err) {
3566 fprintf(stderr, "Could not create the network tree\n");
3567 exit(EXIT_FAILURE);
3568 }
3569 +#endif
3570 +
3571 + struct in6_addr address;
3572 + err = inet_pton(AF_INET6, "2001:db8::1", &address);
3573 + if (err != 1) {
3574 + fprintf(stderr, "Could not parse IP address\n");
3575 + exit(EXIT_FAILURE);
3576 + }
3577
3578 // Create a network
3579 struct loc_network* network1;
3580 @@ -58,12 +68,14 @@ int main(int argc, char** argv) {
3581 exit(EXIT_FAILURE);
3582 }
3583
3584 +#if 0
3585 // Adding network to the tree
3586 err = loc_network_tree_add_network(tree, network1);
3587 if (err) {
3588 fprintf(stderr, "Could not add network to the tree\n");
3589 exit(EXIT_FAILURE);
3590 }
3591 +#endif
3592
3593 // Check if the first and last addresses are correct
3594 char* string = loc_network_format_first_address(network1);
3595 @@ -88,6 +100,12 @@ int main(int argc, char** argv) {
3596 exit(EXIT_FAILURE);
3597 }
3598
3599 + err = loc_network_match_address(network1, &address);
3600 + if (!err) {
3601 + fprintf(stderr, "Network1 does not match address\n");
3602 + exit(EXIT_FAILURE);
3603 + }
3604 +
3605 struct loc_network* network2;
3606 err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48");
3607 if (err) {
3608 @@ -101,6 +119,7 @@ int main(int argc, char** argv) {
3609 exit(EXIT_FAILURE);
3610 }
3611
3612 +#if 0
3613 // Adding network to the tree
3614 err = loc_network_tree_add_network(tree, network2);
3615 if (err) {
3616 @@ -117,20 +136,84 @@ int main(int argc, char** argv) {
3617
3618 size_t nodes = loc_network_tree_count_nodes(tree);
3619 printf("The tree has %zu nodes\n", nodes);
3620 +#endif
3621 +
3622 + // Check equals function
3623 + err = loc_network_cmp(network1, network1);
3624 + if (err) {
3625 + fprintf(stderr, "Network is not equal with itself\n");
3626 + exit(EXIT_FAILURE);
3627 + }
3628 +
3629 + err = loc_network_cmp(network1, network2);
3630 + if (!err) {
3631 + fprintf(stderr, "Networks equal unexpectedly\n");
3632 + exit(EXIT_FAILURE);
3633 + }
3634
3635 // Check subnet function
3636 - err = loc_network_is_subnet_of(network1, network2);
3637 - if (err != 0) {
3638 + err = loc_network_is_subnet(network1, network2);
3639 + if (!err) {
3640 fprintf(stderr, "Subnet check 1 failed: %d\n", err);
3641 exit(EXIT_FAILURE);
3642 }
3643
3644 - err = loc_network_is_subnet_of(network2, network1);
3645 - if (err != 1) {
3646 + err = loc_network_is_subnet(network2, network1);
3647 + if (err) {
3648 fprintf(stderr, "Subnet check 2 failed: %d\n", err);
3649 exit(EXIT_FAILURE);
3650 }
3651
3652 + // Make subnets
3653 + struct loc_network* subnet1 = NULL;
3654 + struct loc_network* subnet2 = NULL;
3655 +
3656 + err = loc_network_subnets(network1, &subnet1, &subnet2);
3657 + if (err || !subnet1 || !subnet2) {
3658 + fprintf(stderr, "Could not find subnets of network: %d\n", err);
3659 + exit(EXIT_FAILURE);
3660 + }
3661 +
3662 + char* s = loc_network_str(subnet1);
3663 + printf("Received subnet1 = %s\n", s);
3664 + free(s);
3665 +
3666 + s = loc_network_str(subnet2);
3667 + printf("Received subnet2 = %s\n", s);
3668 + free(s);
3669 +
3670 + if (!loc_network_is_subnet(network1, subnet1)) {
3671 + fprintf(stderr, "Subnet1 is not a subnet\n");
3672 + exit(EXIT_FAILURE);
3673 + }
3674 +
3675 + if (!loc_network_is_subnet(network1, subnet2)) {
3676 + fprintf(stderr, "Subnet2 is not a subnet\n");
3677 + exit(EXIT_FAILURE);
3678 + }
3679 +
3680 + if (!loc_network_overlaps(network1, subnet1)) {
3681 + fprintf(stderr, "Network1 does not seem to contain subnet1\n");
3682 + exit(EXIT_FAILURE);
3683 + }
3684 +
3685 + if (!loc_network_overlaps(network1, subnet2)) {
3686 + fprintf(stderr, "Network1 does not seem to contain subnet2\n");
3687 + exit(EXIT_FAILURE);
3688 + }
3689 +
3690 + loc_network_unref(subnet1);
3691 + loc_network_unref(subnet2);
3692 +
3693 + struct loc_network_list* excluded = loc_network_exclude(network1, network2);
3694 + if (!excluded) {
3695 + fprintf(stderr, "Could not create excluded list\n");
3696 + exit(EXIT_FAILURE);
3697 + }
3698 +
3699 + loc_network_list_dump(excluded);
3700 + loc_network_list_unref(excluded);
3701 +
3702 // Create a database
3703 struct loc_writer* writer;
3704 err = loc_writer_new(ctx, &writer, NULL, NULL);
3705 @@ -160,6 +243,28 @@ int main(int argc, char** argv) {
3706 // Set ASN
3707 loc_network_set_asn(network4, 1024);
3708
3709 + // Try adding an invalid network
3710 + struct loc_network* network;
3711 + err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
3712 + if (err != -EINVAL) {
3713 + fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
3714 + exit(EXIT_FAILURE);
3715 + }
3716 +
3717 + // Try adding a single address
3718 + err = loc_writer_add_network(writer, &network, "2001:db8::");
3719 + if (err) {
3720 + fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
3721 + exit(EXIT_FAILURE);
3722 + }
3723 +
3724 + // Try adding localhost
3725 + err = loc_writer_add_network(writer, &network, "::1/128");
3726 + if (err != -EINVAL) {
3727 + fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
3728 + exit(EXIT_FAILURE);
3729 + }
3730 +
3731 FILE* f = tmpfile();
3732 if (!f) {
3733 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
3734 @@ -177,7 +282,10 @@ int main(int argc, char** argv) {
3735 loc_network_unref(network2);
3736 loc_network_unref(network3);
3737 loc_network_unref(network4);
3738 +
3739 +#if 0
3740 loc_network_tree_unref(tree);
3741 +#endif
3742
3743 // And open it again from disk
3744 struct loc_database* db;
3745 diff --git a/src/writer.c b/src/writer.c
3746 index 5939cff..2f09b56 100644
3747 --- a/src/writer.c
3748 +++ b/src/writer.c
3749 @@ -147,8 +147,19 @@ static void loc_writer_free(struct loc_writer* writer) {
3750 EVP_PKEY_free(writer->private_key2);
3751
3752 // Unref all AS
3753 - for (unsigned int i = 0; i < writer->as_count; i++) {
3754 - loc_as_unref(writer->as[i]);
3755 + if (writer->as) {
3756 + for (unsigned int i = 0; i < writer->as_count; i++) {
3757 + loc_as_unref(writer->as[i]);
3758 + }
3759 + free(writer->as);
3760 + }
3761 +
3762 + // Unref all countries
3763 + if (writer->countries) {
3764 + for (unsigned int i = 0; i < writer->countries_count; i++) {
3765 + loc_country_unref(writer->countries[i]);
3766 + }
3767 + free(writer->countries);
3768 }
3769
3770 // Release network tree
Peter's tree of IPFire 2.x