1 From ee6ea3986dc80183157f67275dc9f28231b5d5b2 Mon Sep 17 00:00:00 2001
2 From: Michael Tremer <michael.tremer@ipfire.org>
3 Date: Thu, 24 Sep 2020 10:17:58 +0000
4 Subject: [PATCH 01/70] Revert "importer: Purge any redundant entries"
6 This reverts commit c2cc55d5a6875c3838f060032eaed89dcfb92ef6.
8 The query stalls the database and therefore the automatic
9 scripts are no longer able to generate a new version of the
12 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
14 src/python/location-importer.in | 22 +---------------------
15 1 file changed, 1 insertion(+), 21 deletions(-)
17 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
18 index 1467923..e3a07a0 100644
19 --- a/src/python/location-importer.in
20 +++ b/src/python/location-importer.in
21 @@ -374,27 +374,7 @@ class CLI(object):
22 INSERT INTO autnums(number, name)
23 SELECT _autnums.number, _organizations.name FROM _autnums
24 JOIN _organizations ON _autnums.organization = _organizations.handle
25 - ON CONFLICT (number) DO UPDATE SET name = excluded.name
29 - --- Purge any redundant entries
30 - CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
32 - SELECT network FROM networks candidates
34 - SELECT FROM networks
36 - networks.network << candidates.network
38 - networks.country = candidates.country
41 - CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
43 - DELETE FROM networks WHERE EXISTS (
44 - SELECT FROM _garbage WHERE networks.network = _garbage.network
46 + ON CONFLICT (number) DO UPDATE SET name = excluded.name;
49 # Download all extended sources
53 From 92f6abf4e272672bb0a71cfe991261b95ebe2fef Mon Sep 17 00:00:00 2001
54 From: Michael Tremer <michael.tremer@ipfire.org>
55 Date: Thu, 24 Sep 2020 10:18:58 +0000
56 Subject: [PATCH 02/70] Revert "importer: Import raw sources for inetnum's
59 This reverts commit 64e95fa903edec8b4e4e59830b395e2e4a411853.
61 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
63 src/python/importer.py | 14 ++++----
64 src/python/location-importer.in | 63 ---------------------------------
65 2 files changed, 7 insertions(+), 70 deletions(-)
67 diff --git a/src/python/importer.py b/src/python/importer.py
68 index f19db4b..de20f37 100644
69 --- a/src/python/importer.py
70 +++ b/src/python/importer.py
71 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
72 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
74 # Asia Pacific Network Information Centre
75 - "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
76 - "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
77 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
78 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
79 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
80 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
81 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
82 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
85 # Réseaux IP Européens
86 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
87 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
88 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
89 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
90 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
91 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
92 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
93 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
96 # African Network Information Centre
97 - #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
98 + "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
100 # Asia Pacific Network Information Centre
101 - #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
102 + "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
104 # American Registry for Internet Numbers
105 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
106 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
107 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
109 # Réseaux IP Européens
110 - #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
111 + "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
114 class Downloader(object):
115 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
116 index e3a07a0..77952f2 100644
117 --- a/src/python/location-importer.in
118 +++ b/src/python/location-importer.in
119 @@ -393,10 +393,6 @@ class CLI(object):
120 if line.startswith("aut-num:"):
121 return self._parse_autnum_block(block)
124 - if line.startswith("inet6num:") or line.startswith("inetnum:"):
125 - return self._parse_inetnum_block(block)
128 elif line.startswith("organisation:"):
129 return self._parse_org_block(block)
130 @@ -426,65 +422,6 @@ class CLI(object):
131 autnum.get("asn"), autnum.get("org"),
134 - def _parse_inetnum_block(self, block):
135 - logging.debug("Parsing inetnum block:")
139 - logging.debug(line)
142 - key, val = split_line(line)
144 - if key == "inetnum":
145 - start_address, delim, end_address = val.partition("-")
147 - # Strip any excess space
148 - start_address, end_address = start_address.rstrip(), end_address.strip()
150 - # Convert to IP address
152 - start_address = ipaddress.ip_address(start_address)
153 - end_address = ipaddress.ip_address(end_address)
155 - logging.warning("Could not parse line: %s" % line)
158 - # Set prefix to default
161 - # Count number of addresses in this subnet
162 - num_addresses = int(end_address) - int(start_address)
164 - prefix -= math.log(num_addresses, 2)
166 - inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
168 - elif key == "inet6num":
171 - elif key == "country":
172 - if val == "UNITED STATES":
175 - inetnum[key] = val.upper()
177 - # Skip empty objects
181 - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
183 - # Bail out in case we have processed a non-public IP network
184 - if network.is_private:
185 - logging.warning("Skipping non-globally routable network: %s" % network)
188 - self.db.execute("INSERT INTO networks(network, country) \
189 - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
190 - "%s" % network, inetnum.get("country"),
193 def _parse_org_block(self, block):
199 From f532841e9197ce2f40aad8c086d786b2cb783a54 Mon Sep 17 00:00:00 2001
200 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
201 Date: Mon, 12 Oct 2020 20:53:31 +0000
202 Subject: [PATCH 03/70] Revert "Revert "importer: Import raw sources for
205 This reverts commit 92f6abf4e272672bb0a71cfe991261b95ebe2fef.
207 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
209 src/python/importer.py | 14 ++++----
210 src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
211 2 files changed, 70 insertions(+), 7 deletions(-)
213 diff --git a/src/python/importer.py b/src/python/importer.py
214 index de20f37..f19db4b 100644
215 --- a/src/python/importer.py
216 +++ b/src/python/importer.py
217 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
218 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
220 # Asia Pacific Network Information Centre
221 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
222 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
223 + "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
224 + "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
225 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
226 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
227 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
228 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
231 # Réseaux IP Européens
232 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
233 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
234 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
235 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
236 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
237 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
238 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
239 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
242 # African Network Information Centre
243 - "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
244 + #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
246 # Asia Pacific Network Information Centre
247 - "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
248 + #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
250 # American Registry for Internet Numbers
251 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
252 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
253 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
255 # Réseaux IP Européens
256 - "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
257 + #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
260 class Downloader(object):
261 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
262 index 77952f2..e3a07a0 100644
263 --- a/src/python/location-importer.in
264 +++ b/src/python/location-importer.in
265 @@ -393,6 +393,10 @@ class CLI(object):
266 if line.startswith("aut-num:"):
267 return self._parse_autnum_block(block)
270 + if line.startswith("inet6num:") or line.startswith("inetnum:"):
271 + return self._parse_inetnum_block(block)
274 elif line.startswith("organisation:"):
275 return self._parse_org_block(block)
276 @@ -422,6 +426,65 @@ class CLI(object):
277 autnum.get("asn"), autnum.get("org"),
280 + def _parse_inetnum_block(self, block):
281 + logging.debug("Parsing inetnum block:")
285 + logging.debug(line)
288 + key, val = split_line(line)
290 + if key == "inetnum":
291 + start_address, delim, end_address = val.partition("-")
293 + # Strip any excess space
294 + start_address, end_address = start_address.rstrip(), end_address.strip()
296 + # Convert to IP address
298 + start_address = ipaddress.ip_address(start_address)
299 + end_address = ipaddress.ip_address(end_address)
301 + logging.warning("Could not parse line: %s" % line)
304 + # Set prefix to default
307 + # Count number of addresses in this subnet
308 + num_addresses = int(end_address) - int(start_address)
310 + prefix -= math.log(num_addresses, 2)
312 + inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
314 + elif key == "inet6num":
317 + elif key == "country":
318 + if val == "UNITED STATES":
321 + inetnum[key] = val.upper()
323 + # Skip empty objects
327 + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
329 + # Bail out in case we have processed a non-public IP network
330 + if network.is_private:
331 + logging.warning("Skipping non-globally routable network: %s" % network)
334 + self.db.execute("INSERT INTO networks(network, country) \
335 + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
336 + "%s" % network, inetnum.get("country"),
339 def _parse_org_block(self, block):
345 From a36bc686865fc87ea386fd90b389338bdcb80cbc Mon Sep 17 00:00:00 2001
346 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
347 Date: Mon, 12 Oct 2020 20:53:32 +0000
348 Subject: [PATCH 04/70] location-importer.in: only import relevant data from
349 AFRINIC, APNIC and RIPE
351 Content-Type: text/plain; charset=UTF-8
352 Content-Transfer-Encoding: 8bit
354 In contrast to ARIN and LACNIC, we are able to process more detailled
355 feeds from those RIRs, avoiding storage of obviously unnecessary data.
357 Thanks to various SQL optimisations, doing so now takes less time than
358 the first version of this did.
360 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
361 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
362 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
364 src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
365 1 file changed, 87 insertions(+), 2 deletions(-)
367 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
368 index e3a07a0..093f325 100644
369 --- a/src/python/location-importer.in
370 +++ b/src/python/location-importer.in
371 @@ -165,6 +165,7 @@ class CLI(object):
373 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
374 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
375 + CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
376 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
379 @@ -363,6 +364,16 @@ class CLI(object):
380 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
382 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
384 + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
386 + CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
387 + CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
390 + # Remove all previously imported content
391 + self.db.execute("""
392 + TRUNCATE TABLE networks;
395 for source in location.importer.WHOIS_SOURCES:
396 @@ -370,6 +381,67 @@ class CLI(object):
398 self._parse_block(block)
400 + # Process all parsed networks from every RIR we happen to have access to,
401 + # insert the largest network chunks into the networks table immediately...
402 + families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
404 + for family in (row.family for row in families):
405 + smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
407 + self.db.execute("INSERT INTO networks(network, country) \
408 + SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
410 + # ... determine any other prefixes for this network family, ...
411 + prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
412 + WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
414 + # ... and insert networks with this prefix in case they provide additional
415 + # information (i. e. subnet of a larger chunk with a different country)
416 + for prefix in (row.prefix for row in prefixes):
417 + self.db.execute("""
418 + WITH candidates AS (
425 + family(_rirdata.network) = %s
427 + masklen(_rirdata.network) = %s
431 + DISTINCT ON (c.network)
434 + masklen(networks.network),
435 + networks.country AS parent_country
441 + c.network << networks.network
444 + masklen(networks.network) DESC NULLS LAST
447 + networks(network, country)
454 + parent_country IS NULL
456 + country <> parent_country
457 + ON CONFLICT DO NOTHING""",
462 INSERT INTO autnums(number, name)
463 SELECT _autnums.number, _organizations.name FROM _autnums
464 @@ -470,17 +542,30 @@ class CLI(object):
465 inetnum[key] = val.upper()
469 + if not inetnum or not "country" in inetnum:
472 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
474 + # Bail out in case we have processed a network covering the entire IP range, which
475 + # is necessary to work around faulty (?) IPv6 network processing
476 + if network.prefixlen == 0:
477 + logging.warning("Skipping network covering the entire IP adress range: %s" % network)
480 + # Bail out in case we have processed a network whose prefix length indicates it is
481 + # not globally routable (we have decided not to process them at the moment, as they
482 + # significantly enlarge our database without providing very helpful additional information)
483 + if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
484 + logging.info("Skipping network too small to be publicly announced: %s" % network)
487 # Bail out in case we have processed a non-public IP network
488 if network.is_private:
489 logging.warning("Skipping non-globally routable network: %s" % network)
492 - self.db.execute("INSERT INTO networks(network, country) \
493 + self.db.execute("INSERT INTO _rirdata(network, country) \
494 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
495 "%s" % network, inetnum.get("country"),
500 From 2373de384f10f5573bbd7570f5522545df70c0e3 Mon Sep 17 00:00:00 2001
501 From: Michael Tremer <michael.tremer@ipfire.org>
502 Date: Fri, 16 Oct 2020 12:24:58 +0000
503 Subject: [PATCH 05/70] location-importer: Include all overridden networks
505 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
507 src/python/location-importer.in | 2 ++
508 1 file changed, 2 insertions(+)
510 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
511 index 093f325..d249a35 100644
512 --- a/src/python/location-importer.in
513 +++ b/src/python/location-importer.in
514 @@ -240,6 +240,8 @@ class CLI(object):
515 SELECT network FROM announcements
517 SELECT network FROM networks
519 + SELECT network FROM network_overrides
526 From 13f67f285856e8eabfeff2daf1be3aeaa36a82cc Mon Sep 17 00:00:00 2001
527 From: Michael Tremer <michael.tremer@ipfire.org>
528 Date: Fri, 16 Oct 2020 12:26:38 +0000
529 Subject: [PATCH 06/70] Revert "location-importer.in: only import relevant data
530 from AFRINIC, APNIC and RIPE"
532 This reverts commit a36bc686865fc87ea386fd90b389338bdcb80cbc.
534 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
536 src/python/location-importer.in | 89 +--------------------------------
537 1 file changed, 2 insertions(+), 87 deletions(-)
539 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
540 index d249a35..b220eaf 100644
541 --- a/src/python/location-importer.in
542 +++ b/src/python/location-importer.in
543 @@ -165,7 +165,6 @@ class CLI(object):
545 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
546 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
547 - CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
548 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
551 @@ -366,16 +365,6 @@ class CLI(object):
552 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
554 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
556 - CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
558 - CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
559 - CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
562 - # Remove all previously imported content
563 - self.db.execute("""
564 - TRUNCATE TABLE networks;
567 for source in location.importer.WHOIS_SOURCES:
568 @@ -383,67 +372,6 @@ class CLI(object):
570 self._parse_block(block)
572 - # Process all parsed networks from every RIR we happen to have access to,
573 - # insert the largest network chunks into the networks table immediately...
574 - families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
576 - for family in (row.family for row in families):
577 - smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
579 - self.db.execute("INSERT INTO networks(network, country) \
580 - SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
582 - # ... determine any other prefixes for this network family, ...
583 - prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
584 - WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
586 - # ... and insert networks with this prefix in case they provide additional
587 - # information (i. e. subnet of a larger chunk with a different country)
588 - for prefix in (row.prefix for row in prefixes):
589 - self.db.execute("""
590 - WITH candidates AS (
597 - family(_rirdata.network) = %s
599 - masklen(_rirdata.network) = %s
603 - DISTINCT ON (c.network)
606 - masklen(networks.network),
607 - networks.country AS parent_country
613 - c.network << networks.network
616 - masklen(networks.network) DESC NULLS LAST
619 - networks(network, country)
626 - parent_country IS NULL
628 - country <> parent_country
629 - ON CONFLICT DO NOTHING""",
634 INSERT INTO autnums(number, name)
635 SELECT _autnums.number, _organizations.name FROM _autnums
636 @@ -544,30 +472,17 @@ class CLI(object):
637 inetnum[key] = val.upper()
640 - if not inetnum or not "country" in inetnum:
644 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
646 - # Bail out in case we have processed a network covering the entire IP range, which
647 - # is necessary to work around faulty (?) IPv6 network processing
648 - if network.prefixlen == 0:
649 - logging.warning("Skipping network covering the entire IP adress range: %s" % network)
652 - # Bail out in case we have processed a network whose prefix length indicates it is
653 - # not globally routable (we have decided not to process them at the moment, as they
654 - # significantly enlarge our database without providing very helpful additional information)
655 - if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
656 - logging.info("Skipping network too small to be publicly announced: %s" % network)
659 # Bail out in case we have processed a non-public IP network
660 if network.is_private:
661 logging.warning("Skipping non-globally routable network: %s" % network)
664 - self.db.execute("INSERT INTO _rirdata(network, country) \
665 + self.db.execute("INSERT INTO networks(network, country) \
666 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
667 "%s" % network, inetnum.get("country"),
672 From 44341478233115b26bb27fdb24da5b0a1eedb173 Mon Sep 17 00:00:00 2001
673 From: Michael Tremer <michael.tremer@ipfire.org>
674 Date: Fri, 16 Oct 2020 12:26:43 +0000
675 Subject: [PATCH 07/70] Revert "Revert "Revert "importer: Import raw sources
676 for inetnum's again"""
678 This reverts commit f532841e9197ce2f40aad8c086d786b2cb783a54.
680 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
682 src/python/importer.py | 14 ++++----
683 src/python/location-importer.in | 63 ---------------------------------
684 2 files changed, 7 insertions(+), 70 deletions(-)
686 diff --git a/src/python/importer.py b/src/python/importer.py
687 index f19db4b..de20f37 100644
688 --- a/src/python/importer.py
689 +++ b/src/python/importer.py
690 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
691 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
693 # Asia Pacific Network Information Centre
694 - "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
695 - "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
696 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
697 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
698 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
699 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
700 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
701 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
704 # Réseaux IP Européens
705 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
706 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
707 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
708 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
709 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
710 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
711 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
712 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
715 # African Network Information Centre
716 - #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
717 + "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
719 # Asia Pacific Network Information Centre
720 - #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
721 + "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
723 # American Registry for Internet Numbers
724 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
725 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
726 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
728 # Réseaux IP Européens
729 - #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
730 + "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
733 class Downloader(object):
734 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
735 index b220eaf..e87d378 100644
736 --- a/src/python/location-importer.in
737 +++ b/src/python/location-importer.in
738 @@ -395,10 +395,6 @@ class CLI(object):
739 if line.startswith("aut-num:"):
740 return self._parse_autnum_block(block)
743 - if line.startswith("inet6num:") or line.startswith("inetnum:"):
744 - return self._parse_inetnum_block(block)
747 elif line.startswith("organisation:"):
748 return self._parse_org_block(block)
749 @@ -428,65 +424,6 @@ class CLI(object):
750 autnum.get("asn"), autnum.get("org"),
753 - def _parse_inetnum_block(self, block):
754 - logging.debug("Parsing inetnum block:")
758 - logging.debug(line)
761 - key, val = split_line(line)
763 - if key == "inetnum":
764 - start_address, delim, end_address = val.partition("-")
766 - # Strip any excess space
767 - start_address, end_address = start_address.rstrip(), end_address.strip()
769 - # Convert to IP address
771 - start_address = ipaddress.ip_address(start_address)
772 - end_address = ipaddress.ip_address(end_address)
774 - logging.warning("Could not parse line: %s" % line)
777 - # Set prefix to default
780 - # Count number of addresses in this subnet
781 - num_addresses = int(end_address) - int(start_address)
783 - prefix -= math.log(num_addresses, 2)
785 - inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
787 - elif key == "inet6num":
790 - elif key == "country":
791 - if val == "UNITED STATES":
794 - inetnum[key] = val.upper()
796 - # Skip empty objects
800 - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
802 - # Bail out in case we have processed a non-public IP network
803 - if network.is_private:
804 - logging.warning("Skipping non-globally routable network: %s" % network)
807 - self.db.execute("INSERT INTO networks(network, country) \
808 - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
809 - "%s" % network, inetnum.get("country"),
812 def _parse_org_block(self, block):
818 From a7d3a7a0565a0e09d3442e5829a0f30f016993b9 Mon Sep 17 00:00:00 2001
819 From: Michael Tremer <michael.tremer@ipfire.org>
820 Date: Tue, 20 Oct 2020 20:44:43 +0000
821 Subject: [PATCH 08/70] as: Fix dereferencing NULL pointer when setting AS name
823 Reported-by: Gisle Vanem <gisle.vanem@gmail.com>
824 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
826 src/as.c | 8 +++++++-
827 1 file changed, 7 insertions(+), 1 deletion(-)
829 diff --git a/src/as.c b/src/as.c
830 index e1fbb01..8421ac8 100644
833 @@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
836 LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) {
837 - as->name = strdup(name);
842 + as->name = strdup(name);
851 From ddb326ad38a7c7202315dd2c6f938313db04ee22 Mon Sep 17 00:00:00 2001
852 From: Michael Tremer <michael.tremer@ipfire.org>
853 Date: Wed, 21 Oct 2020 09:18:08 +0000
854 Subject: [PATCH 09/70] as: Do not attempt to match name when it wasn't set
856 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
859 1 file changed, 4 insertions(+)
861 diff --git a/src/as.c b/src/as.c
862 index 8421ac8..757bf3d 100644
865 @@ -145,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
869 + // Cannot match anything when name is not set
873 // Search if string is in name
874 if (strcasestr(as->name, string) != NULL)
879 From d226ad2d97cbcd42ce807d9308569b1b9c5d4e2f Mon Sep 17 00:00:00 2001
880 From: Michael Tremer <michael.tremer@ipfire.org>
881 Date: Wed, 21 Oct 2020 09:28:39 +0000
882 Subject: [PATCH 10/70] writer: Free array with pointer to ASes, too
884 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
886 src/writer.c | 7 +++++--
887 1 file changed, 5 insertions(+), 2 deletions(-)
889 diff --git a/src/writer.c b/src/writer.c
890 index 5939cff..160650f 100644
893 @@ -147,8 +147,11 @@ static void loc_writer_free(struct loc_writer* writer) {
894 EVP_PKEY_free(writer->private_key2);
897 - for (unsigned int i = 0; i < writer->as_count; i++) {
898 - loc_as_unref(writer->as[i]);
900 + for (unsigned int i = 0; i < writer->as_count; i++) {
901 + loc_as_unref(writer->as[i]);
906 // Release network tree
910 From d89a7d62772048ae1bd18d03f69df46b7e1a3d3c Mon Sep 17 00:00:00 2001
911 From: Michael Tremer <michael.tremer@ipfire.org>
912 Date: Wed, 21 Oct 2020 09:31:29 +0000
913 Subject: [PATCH 11/70] writer: Free countries when the writer is being
916 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
918 src/writer.c | 8 ++++++++
919 1 file changed, 8 insertions(+)
921 diff --git a/src/writer.c b/src/writer.c
922 index 160650f..2f09b56 100644
925 @@ -154,6 +154,14 @@ static void loc_writer_free(struct loc_writer* writer) {
929 + // Unref all countries
930 + if (writer->countries) {
931 + for (unsigned int i = 0; i < writer->countries_count; i++) {
932 + loc_country_unref(writer->countries[i]);
934 + free(writer->countries);
937 // Release network tree
938 if (writer->networks)
939 loc_network_tree_unref(writer->networks);
943 From 0f1aedbc68e3945770c93e0ebd83eed0f555d6f0 Mon Sep 17 00:00:00 2001
944 From: Michael Tremer <michael.tremer@ipfire.org>
945 Date: Wed, 21 Oct 2020 13:19:44 +0000
946 Subject: [PATCH 12/70] tests: Try adding an invalid network
948 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
950 src/test-network.c | 8 ++++++++
951 1 file changed, 8 insertions(+)
953 diff --git a/src/test-network.c b/src/test-network.c
954 index d38f13d..e908b57 100644
955 --- a/src/test-network.c
956 +++ b/src/test-network.c
957 @@ -160,6 +160,14 @@ int main(int argc, char** argv) {
959 loc_network_set_asn(network4, 1024);
961 + // Try adding an invalid network
962 + struct loc_network* network;
963 + err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
964 + if (err != -EINVAL) {
965 + fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
966 + exit(EXIT_FAILURE);
971 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
975 From 13ad6e695f9ffc7847b3afe3e9cbcea8fb3a443f Mon Sep 17 00:00:00 2001
976 From: Michael Tremer <michael.tremer@ipfire.org>
977 Date: Wed, 21 Oct 2020 13:36:35 +0000
978 Subject: [PATCH 13/70] networks: Improve parsing IP addresses
980 loc_network_new_from_string() seem to have had some unexpected
981 behaviour for invalid inputs.
983 The function has been tidied up slightly and returns as soon as
984 some invalid input was detected.
986 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
988 src/network.c | 52 +++++++++++++++++++++++++++++++---------------
989 src/test-network.c | 7 +++++++
990 2 files changed, 42 insertions(+), 17 deletions(-)
992 diff --git a/src/network.c b/src/network.c
993 index 366caa2..c112a41 100644
996 @@ -160,9 +160,10 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
997 LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
998 const char* address_string) {
999 struct in6_addr first_address;
1000 - unsigned int prefix = 0;
1001 char* prefix_string;
1005 + DEBUG(ctx, "Attempting to parse network %s\n", address_string);
1007 // Make a copy of the string to work on it
1008 char* buffer = strdup(address_string);
1009 @@ -171,29 +172,46 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1010 // Split address and prefix
1011 address_string = strsep(&prefix_string, "/");
1013 - // Did we find a prefix?
1014 - if (prefix_string) {
1015 - // Convert prefix to integer
1016 - prefix = strtol(prefix_string, NULL, 10);
1017 + DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
1020 - // Parse the address
1021 - r = loc_parse_address(ctx, address_string, &first_address);
1022 + // We need to have a prefix
1023 + if (!prefix_string) {
1024 + DEBUG(ctx, "No prefix set\n");
1028 - // Map the prefix to IPv6 if needed
1029 - if (IN6_IS_ADDR_V4MAPPED(&first_address))
1032 + // Convert prefix to integer
1033 + unsigned int prefix = strtol(prefix_string, NULL, 10);
1035 + // End if the prefix was invalid
1037 + DEBUG(ctx, "The prefix is zero or not a number\n");
1041 + // Parse the address
1042 + r = loc_parse_address(ctx, address_string, &first_address);
1044 + DEBUG(ctx, "The address could not be parsed\n");
1048 + // Map the prefix to IPv6 if needed
1049 + if (IN6_IS_ADDR_V4MAPPED(&first_address))
1053 // Free temporary buffer
1057 - r = loc_network_new(ctx, network, &first_address, prefix);
1059 + // Exit if the parsing was unsuccessful
1063 + DEBUG(ctx, "GOT HERE\n");
1066 + // Create a new network
1067 + return loc_network_new(ctx, network, &first_address, prefix);
1070 LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
1071 diff --git a/src/test-network.c b/src/test-network.c
1072 index e908b57..85eca00 100644
1073 --- a/src/test-network.c
1074 +++ b/src/test-network.c
1075 @@ -168,6 +168,13 @@ int main(int argc, char** argv) {
1079 + // Try adding a single address
1080 + err = loc_writer_add_network(writer, &network, "2001:db8::");
1081 + if (err != -EINVAL) {
1082 + fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
1083 + exit(EXIT_FAILURE);
1086 FILE* f = tmpfile();
1088 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
1092 From 6a467e9345bb5a3d37911c9aaac30019eaa4492b Mon Sep 17 00:00:00 2001
1093 From: Michael Tremer <michael.tremer@ipfire.org>
1094 Date: Wed, 21 Oct 2020 13:43:21 +0000
1095 Subject: [PATCH 14/70] networks: Test if we can add localhost (IPv6)
1097 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1099 src/test-network.c | 7 +++++++
1100 1 file changed, 7 insertions(+)
1102 diff --git a/src/test-network.c b/src/test-network.c
1103 index 85eca00..8c7e898 100644
1104 --- a/src/test-network.c
1105 +++ b/src/test-network.c
1106 @@ -175,6 +175,13 @@ int main(int argc, char** argv) {
1110 + // Try adding localhost
1111 + err = loc_writer_add_network(writer, &network, "::1/128");
1112 + if (err != -EINVAL) {
1113 + fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
1114 + exit(EXIT_FAILURE);
1117 FILE* f = tmpfile();
1119 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
1123 From fc1190aa11e3ff3d2dbf5f4d408c298e7916f46f Mon Sep 17 00:00:00 2001
1124 From: Michael Tremer <michael.tremer@ipfire.org>
1125 Date: Wed, 21 Oct 2020 13:44:50 +0000
1126 Subject: [PATCH 15/70] networks: Remove accidentially committed debug line
1128 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1130 src/network.c | 2 --
1131 1 file changed, 2 deletions(-)
1133 diff --git a/src/network.c b/src/network.c
1134 index c112a41..be88d75 100644
1137 @@ -208,8 +208,6 @@ FAIL:
1141 - DEBUG(ctx, "GOT HERE\n");
1143 // Create a new network
1144 return loc_network_new(ctx, network, &first_address, prefix);
1149 From a1707d8983898b6878cdd5c68744bcc444e278ed Mon Sep 17 00:00:00 2001
1150 From: Michael Tremer <michael.tremer@ipfire.org>
1151 Date: Wed, 21 Oct 2020 13:53:36 +0000
1152 Subject: [PATCH 16/70] importer: Add search index to announcements table
1154 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1156 src/python/location-importer.in | 1 +
1157 1 file changed, 1 insertion(+)
1159 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1160 index e87d378..d0fe5a6 100644
1161 --- a/src/python/location-importer.in
1162 +++ b/src/python/location-importer.in
1163 @@ -152,6 +152,7 @@ class CLI(object):
1164 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
1165 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
1166 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
1167 + CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
1170 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
1174 From 991baf530d47adb2ed7a15b65dc4565d07fa6d07 Mon Sep 17 00:00:00 2001
1175 From: Michael Tremer <michael.tremer@ipfire.org>
1176 Date: Wed, 21 Oct 2020 13:54:45 +0000
1177 Subject: [PATCH 17/70] importer: Add search index to network_overrides table
1179 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1181 src/python/location-importer.in | 2 ++
1182 1 file changed, 2 insertions(+)
1184 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1185 index d0fe5a6..fe21d73 100644
1186 --- a/src/python/location-importer.in
1187 +++ b/src/python/location-importer.in
1188 @@ -189,6 +189,8 @@ class CLI(object):
1190 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
1191 ON network_overrides(network);
1192 + CREATE INDEX IF NOT EXISTS network_overrides_search
1193 + ON network_overrides USING GIST(network inet_ops);
1200 From bbea93a74651df10e2ffdbd09eb434dc6a0471bc Mon Sep 17 00:00:00 2001
1201 From: Michael Tremer <michael.tremer@ipfire.org>
1202 Date: Wed, 21 Oct 2020 16:01:57 +0000
1203 Subject: [PATCH 18/70] importer: Restructure SQL query to be executed in
1206 There are no functional changes, this just runs quicker now.
1208 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1210 src/python/location-importer.in | 67 ++++++++++++++++++---------------
1211 1 file changed, 37 insertions(+), 30 deletions(-)
1213 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1214 index fe21d73..3c1e5e2 100644
1215 --- a/src/python/location-importer.in
1216 +++ b/src/python/location-importer.in
1217 @@ -237,34 +237,24 @@ class CLI(object):
1219 # Select all known networks
1220 rows = self.db.query("""
1221 - -- Get a (sorted) list of all known networks
1222 - WITH known_networks AS (
1223 - SELECT network FROM announcements
1225 - SELECT network FROM networks
1227 - SELECT network FROM network_overrides
1231 -- Return a list of those networks enriched with all
1232 -- other information that we store in the database
1234 - DISTINCT ON (known_networks.network)
1235 - known_networks.network AS network,
1236 - announcements.autnum AS autnum,
1237 + DISTINCT ON (network)
1244 SELECT country FROM network_overrides overrides
1245 - WHERE announcements.network <<= overrides.network
1246 + WHERE networks.network <<= overrides.network
1247 ORDER BY masklen(overrides.network) DESC
1251 SELECT country FROM autnum_overrides overrides
1252 - WHERE announcements.autnum = overrides.number
1253 + WHERE networks.autnum = overrides.number
1257 @@ -273,50 +263,67 @@ class CLI(object):
1260 SELECT is_anonymous_proxy FROM network_overrides overrides
1261 - WHERE announcements.network <<= overrides.network
1262 + WHERE networks.network <<= overrides.network
1263 ORDER BY masklen(overrides.network) DESC
1267 SELECT is_anonymous_proxy FROM autnum_overrides overrides
1268 - WHERE announcements.autnum = overrides.number
1269 + WHERE networks.autnum = overrides.number
1272 ) AS is_anonymous_proxy,
1275 SELECT is_satellite_provider FROM network_overrides overrides
1276 - WHERE announcements.network <<= overrides.network
1277 + WHERE networks.network <<= overrides.network
1278 ORDER BY masklen(overrides.network) DESC
1282 SELECT is_satellite_provider FROM autnum_overrides overrides
1283 - WHERE announcements.autnum = overrides.number
1284 + WHERE networks.autnum = overrides.number
1287 ) AS is_satellite_provider,
1290 SELECT is_anycast FROM network_overrides overrides
1291 - WHERE announcements.network <<= overrides.network
1292 + WHERE networks.network <<= overrides.network
1293 ORDER BY masklen(overrides.network) DESC
1297 SELECT is_anycast FROM autnum_overrides overrides
1298 - WHERE announcements.autnum = overrides.number
1299 + WHERE networks.autnum = overrides.number
1304 - -- Must be part of returned values for ORDER BY clause
1305 - masklen(announcements.network) AS sort_a,
1306 - masklen(networks.network) AS sort_b
1307 - FROM known_networks
1308 - LEFT JOIN announcements ON known_networks.network <<= announcements.network
1309 - LEFT JOIN networks ON known_networks.network <<= networks.network
1310 - ORDER BY known_networks.network, sort_a DESC, sort_b DESC
1314 + known_networks.network AS network,
1315 + announcements.autnum AS autnum,
1316 + networks.country AS country,
1318 + -- Must be part of returned values for ORDER BY clause
1319 + masklen(announcements.network) AS sort_a,
1320 + masklen(networks.network) AS sort_b
1322 + SELECT network FROM announcements
1324 + SELECT network FROM networks
1326 + SELECT network FROM network_overrides
1329 + announcements ON known_networks.network <<= announcements.network
1331 + networks ON known_networks.network <<= networks.network
1333 + known_networks.network,
1343 From 26ab419b68d166f932db1f97c38cb9d793d04187 Mon Sep 17 00:00:00 2001
1344 From: Michael Tremer <michael.tremer@ipfire.org>
1345 Date: Thu, 22 Oct 2020 12:24:34 +0000
1346 Subject: [PATCH 19/70] network: Allow adding single IP addresses and
1347 automatically add the prefix
1349 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1351 src/network.c | 33 +++++++++++++++------------------
1352 src/test-network.c | 4 ++--
1353 2 files changed, 17 insertions(+), 20 deletions(-)
1355 diff --git a/src/network.c b/src/network.c
1356 index be88d75..d7b1645 100644
1359 @@ -161,6 +161,7 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1360 const char* address_string) {
1361 struct in6_addr first_address;
1362 char* prefix_string;
1363 + unsigned int prefix = 128;
1366 DEBUG(ctx, "Attempting to parse network %s\n", address_string);
1367 @@ -174,21 +175,6 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1369 DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
1371 - // We need to have a prefix
1372 - if (!prefix_string) {
1373 - DEBUG(ctx, "No prefix set\n");
1377 - // Convert prefix to integer
1378 - unsigned int prefix = strtol(prefix_string, NULL, 10);
1380 - // End if the prefix was invalid
1382 - DEBUG(ctx, "The prefix is zero or not a number\n");
1386 // Parse the address
1387 r = loc_parse_address(ctx, address_string, &first_address);
1389 @@ -196,9 +182,20 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1393 - // Map the prefix to IPv6 if needed
1394 - if (IN6_IS_ADDR_V4MAPPED(&first_address))
1396 + // If a prefix was given, we will try to parse it
1397 + if (prefix_string) {
1398 + // Convert prefix to integer
1399 + prefix = strtol(prefix_string, NULL, 10);
1402 + DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
1406 + // Map the prefix to IPv6 if needed
1407 + if (IN6_IS_ADDR_V4MAPPED(&first_address))
1412 // Free temporary buffer
1413 diff --git a/src/test-network.c b/src/test-network.c
1414 index 8c7e898..b6776b4 100644
1415 --- a/src/test-network.c
1416 +++ b/src/test-network.c
1417 @@ -170,8 +170,8 @@ int main(int argc, char** argv) {
1419 // Try adding a single address
1420 err = loc_writer_add_network(writer, &network, "2001:db8::");
1421 - if (err != -EINVAL) {
1422 - fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
1424 + fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
1431 From aadac4c569e921be1d28dd3b2377ac7f3732213e Mon Sep 17 00:00:00 2001
1432 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1433 Date: Wed, 21 Oct 2020 14:47:36 +0000
1434 Subject: [PATCH 20/70] Revert "Revert "Revert "Revert "importer: Import raw
1435 sources for inetnum's again""""
1437 Content-Type: text/plain; charset=UTF-8
1438 Content-Transfer-Encoding: 8bit
1440 This reverts commit 44341478233115b26bb27fdb24da5b0a1eedb173.
1442 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1443 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1445 src/python/importer.py | 14 ++++----
1446 src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
1447 2 files changed, 70 insertions(+), 7 deletions(-)
1449 diff --git a/src/python/importer.py b/src/python/importer.py
1450 index de20f37..f19db4b 100644
1451 --- a/src/python/importer.py
1452 +++ b/src/python/importer.py
1453 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
1454 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
1456 # Asia Pacific Network Information Centre
1457 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
1458 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
1459 + "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
1460 + "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
1461 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
1462 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
1463 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
1464 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
1467 # Réseaux IP Européens
1468 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
1469 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
1470 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
1471 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
1472 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
1473 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
1474 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
1475 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
1477 EXTENDED_SOURCES = (
1478 # African Network Information Centre
1479 - "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
1480 + #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
1482 # Asia Pacific Network Information Centre
1483 - "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
1484 + #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
1486 # American Registry for Internet Numbers
1487 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
1488 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
1489 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
1491 # Réseaux IP Européens
1492 - "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
1493 + #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
1496 class Downloader(object):
1497 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1498 index 3c1e5e2..e8a4fc5 100644
1499 --- a/src/python/location-importer.in
1500 +++ b/src/python/location-importer.in
1501 @@ -405,6 +405,10 @@ class CLI(object):
1502 if line.startswith("aut-num:"):
1503 return self._parse_autnum_block(block)
1506 + if line.startswith("inet6num:") or line.startswith("inetnum:"):
1507 + return self._parse_inetnum_block(block)
1510 elif line.startswith("organisation:"):
1511 return self._parse_org_block(block)
1512 @@ -434,6 +438,65 @@ class CLI(object):
1513 autnum.get("asn"), autnum.get("org"),
1516 + def _parse_inetnum_block(self, block):
1517 + logging.debug("Parsing inetnum block:")
1520 + for line in block:
1521 + logging.debug(line)
1524 + key, val = split_line(line)
1526 + if key == "inetnum":
1527 + start_address, delim, end_address = val.partition("-")
1529 + # Strip any excess space
1530 + start_address, end_address = start_address.rstrip(), end_address.strip()
1532 + # Convert to IP address
1534 + start_address = ipaddress.ip_address(start_address)
1535 + end_address = ipaddress.ip_address(end_address)
1536 + except ValueError:
1537 + logging.warning("Could not parse line: %s" % line)
1540 + # Set prefix to default
1543 + # Count number of addresses in this subnet
1544 + num_addresses = int(end_address) - int(start_address)
1546 + prefix -= math.log(num_addresses, 2)
1548 + inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
1550 + elif key == "inet6num":
1551 + inetnum[key] = val
1553 + elif key == "country":
1554 + if val == "UNITED STATES":
1557 + inetnum[key] = val.upper()
1559 + # Skip empty objects
1563 + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
1565 + # Bail out in case we have processed a non-public IP network
1566 + if network.is_private:
1567 + logging.warning("Skipping non-globally routable network: %s" % network)
1570 + self.db.execute("INSERT INTO networks(network, country) \
1571 + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
1572 + "%s" % network, inetnum.get("country"),
1575 def _parse_org_block(self, block):
1581 From 002deb6b42ac0b3624c07e3352cebd72dc0685a2 Mon Sep 17 00:00:00 2001
1582 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1583 Date: Wed, 21 Oct 2020 14:47:37 +0000
1584 Subject: [PATCH 21/70] Revert "Revert "location-importer.in: only import
1585 relevant data from AFRINIC, APNIC and RIPE""
1587 Content-Type: text/plain; charset=UTF-8
1588 Content-Transfer-Encoding: 8bit
1590 This reverts commit 13f67f285856e8eabfeff2daf1be3aeaa36a82cc.
1592 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1593 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1595 src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
1596 1 file changed, 87 insertions(+), 2 deletions(-)
1598 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1599 index e8a4fc5..5656c41 100644
1600 --- a/src/python/location-importer.in
1601 +++ b/src/python/location-importer.in
1602 @@ -166,6 +166,7 @@ class CLI(object):
1604 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
1605 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
1606 + CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
1607 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
1610 @@ -375,6 +376,16 @@ class CLI(object):
1611 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
1613 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
1615 + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
1617 + CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
1618 + CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
1621 + # Remove all previously imported content
1622 + self.db.execute("""
1623 + TRUNCATE TABLE networks;
1626 for source in location.importer.WHOIS_SOURCES:
1627 @@ -382,6 +393,67 @@ class CLI(object):
1629 self._parse_block(block)
1631 + # Process all parsed networks from every RIR we happen to have access to,
1632 + # insert the largest network chunks into the networks table immediately...
1633 + families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
1635 + for family in (row.family for row in families):
1636 + smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
1638 + self.db.execute("INSERT INTO networks(network, country) \
1639 + SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
1641 + # ... determine any other prefixes for this network family, ...
1642 + prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
1643 + WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
1645 + # ... and insert networks with this prefix in case they provide additional
1646 + # information (i. e. subnet of a larger chunk with a different country)
1647 + for prefix in (row.prefix for row in prefixes):
1648 + self.db.execute("""
1649 + WITH candidates AS (
1656 + family(_rirdata.network) = %s
1658 + masklen(_rirdata.network) = %s
1662 + DISTINCT ON (c.network)
1665 + masklen(networks.network),
1666 + networks.country AS parent_country
1672 + c.network << networks.network
1675 + masklen(networks.network) DESC NULLS LAST
1678 + networks(network, country)
1685 + parent_country IS NULL
1687 + country <> parent_country
1688 + ON CONFLICT DO NOTHING""",
1693 INSERT INTO autnums(number, name)
1694 SELECT _autnums.number, _organizations.name FROM _autnums
1695 @@ -482,17 +554,30 @@ class CLI(object):
1696 inetnum[key] = val.upper()
1698 # Skip empty objects
1700 + if not inetnum or not "country" in inetnum:
1703 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
1705 + # Bail out in case we have processed a network covering the entire IP range, which
1706 + # is necessary to work around faulty (?) IPv6 network processing
1707 + if network.prefixlen == 0:
1708 + logging.warning("Skipping network covering the entire IP adress range: %s" % network)
1711 + # Bail out in case we have processed a network whose prefix length indicates it is
1712 + # not globally routable (we have decided not to process them at the moment, as they
1713 + # significantly enlarge our database without providing very helpful additional information)
1714 + if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
1715 + logging.info("Skipping network too small to be publicly announced: %s" % network)
1718 # Bail out in case we have processed a non-public IP network
1719 if network.is_private:
1720 logging.warning("Skipping non-globally routable network: %s" % network)
1723 - self.db.execute("INSERT INTO networks(network, country) \
1724 + self.db.execute("INSERT INTO _rirdata(network, country) \
1725 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
1726 "%s" % network, inetnum.get("country"),
1731 From 28c73fa3f4257e0a41e52af8a9643da414a6cb6f Mon Sep 17 00:00:00 2001
1732 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1733 Date: Wed, 21 Oct 2020 14:47:38 +0000
1734 Subject: [PATCH 22/70] export.py: fix exporting IP networks for crappy
1737 Content-Type: text/plain; charset=UTF-8
1738 Content-Transfer-Encoding: 8bit
1740 In contrast to the location database itself, the xt_geoip module
1741 consumes a list of IP networks for each country, and returns after the
1744 We therefore need to...
1746 (a) sort IP networks by their size, allow as precise matches as possible
1747 (b) export _any_ IP networks - including inverted subnets - to prevent
1749 (c) do the entire thing as fast as possible, consuming as less disk
1750 space as possible, which is why we can't just iterate over all /24
1753 Partially fixes: #12499
1755 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1756 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1757 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1759 src/python/export.py | 69 ++++++++++++++++++++++++++++++++++----------
1760 1 file changed, 54 insertions(+), 15 deletions(-)
1762 diff --git a/src/python/export.py b/src/python/export.py
1763 index d15c6f0..5eaf43f 100644
1764 --- a/src/python/export.py
1765 +++ b/src/python/export.py
1766 @@ -39,8 +39,8 @@ class OutputWriter(object):
1770 - def __init__(self, f, prefix=None, flatten=True):
1771 - self.f, self.prefix, self.flatten = f, prefix, flatten
1772 + def __init__(self, db, f, prefix=None, flatten=True):
1773 + self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten
1775 # The previously written network
1776 self._last_network = None
1777 @@ -49,13 +49,13 @@ class OutputWriter(object):
1778 self._write_header()
1781 - def open(cls, filename, **kwargs):
1782 + def open(cls, db, filename, **kwargs):
1784 Convenience function to open a file
1786 f = open(filename, cls.mode)
1788 - return cls(f, **kwargs)
1789 + return cls(db, f, **kwargs)
1792 return "<%s f=%s>" % (self.__class__.__name__, self.f)
1793 @@ -87,13 +87,31 @@ class OutputWriter(object):
1794 def _write_network(self, network):
1795 self.f.write("%s\n" % network)
1797 - def write(self, network):
1798 + def write(self, network, subnets):
1799 if self.flatten and self._flatten(network):
1800 log.debug("Skipping writing network %s" % network)
1803 - # Write the network to file
1804 - self._write_network(network)
1805 + # Write the network when it has no subnets
1807 + network = ipaddress.ip_network("%s" % network)
1808 + return self._write_network(network)
1810 + # Collect all matching subnets
1811 + matching_subnets = []
1813 + for subnet in sorted(subnets):
1814 + # Try to find the subnet in the database
1815 + n = self.db.lookup("%s" % subnet.network_address)
1817 + # No entry or matching country means those IP addresses belong here
1818 + if not n or n.country_code == network.country_code:
1819 + matching_subnets.append(subnet)
1821 + # Write all networks as compact as possible
1822 + for network in ipaddress.collapse_addresses(matching_subnets):
1823 + log.debug("Writing %s to database" % network)
1824 + self._write_network(network)
1828 @@ -143,10 +161,10 @@ class XTGeoIPOutputWriter(OutputWriter):
1831 def _write_network(self, network):
1832 - for address in (network.first_address, network.last_address):
1833 + for address in (network.network_address, network.broadcast_address):
1834 # Convert this into a string of bits
1835 bytes = socket.inet_pton(
1836 - network.family, address,
1837 + socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
1841 @@ -175,7 +193,7 @@ class Exporter(object):
1842 directory, prefix=country_code, suffix=self.writer.suffix, family=family,
1845 - writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
1846 + writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code)
1848 # Create writers for ASNs
1850 @@ -183,22 +201,43 @@ class Exporter(object):
1851 directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
1854 - writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
1855 + writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
1857 # Get all networks that match the family
1858 networks = self.db.search_networks(family=family)
1860 + # Materialise the generator into a list (uses quite some memory)
1861 + networks = list(networks)
1863 # Walk through all networks
1864 - for network in networks:
1865 + for i, network in enumerate(networks):
1866 + _network = ipaddress.ip_network("%s" % network)
1868 + # Search for all subnets
1871 + while i < len(networks):
1872 + subnet = networks[i+1]
1874 + if subnet.is_subnet_of(network):
1875 + _subnet = ipaddress.ip_network("%s" % subnet)
1877 + subnets.add(_subnet)
1878 + subnets.update(_network.address_exclude(_subnet))
1884 # Write matching countries
1886 - writers[network.country_code].write(network)
1887 + writers[network.country_code].write(network, subnets)
1891 # Write matching ASNs
1893 - writers[network.asn].write(network)
1894 + writers[network.asn].write(network, subnets)
1898 @@ -209,7 +248,7 @@ class Exporter(object):
1899 country = flags[flag]
1902 - writers[country].write(network)
1903 + writers[country].write(network, subnets)
1910 From bd341642fc6bbcc050e9b4ec5124585c83cab84d Mon Sep 17 00:00:00 2001
1911 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1912 Date: Wed, 21 Oct 2020 14:47:39 +0000
1913 Subject: [PATCH 23/70] location-importer.in: filter bogus IP networks for both
1914 Whois and extended sources
1916 Content-Type: text/plain; charset=UTF-8
1917 Content-Transfer-Encoding: 8bit
1919 Sanity checks for parsed networks have been put into a separate function
1920 to avoid boilerplate code for extended sources. This makes the location
1921 database less vulnerable to garbage written into RIR databases on
1922 purpose or by chance.
1926 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1927 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1929 src/python/location-importer.in | 83 ++++++++++++++++++++++++++-------
1930 1 file changed, 67 insertions(+), 16 deletions(-)
1932 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1933 index 5656c41..f24d357 100644
1934 --- a/src/python/location-importer.in
1935 +++ b/src/python/location-importer.in
1936 @@ -469,6 +469,69 @@ class CLI(object):
1938 self._parse_line(line)
1940 + def _check_parsed_network(self, network):
1942 + Assistive function to detect and subsequently sort out parsed
1943 + networks from RIR data (both Whois and so-called "extended sources"),
1944 + which are or have...
1946 + (a) not globally routable (RFC 1918 space, et al.)
1947 + (b) covering a too large chunk of the IP address space (prefix length
1948 + is < 7 for IPv4 networks, and < 10 for IPv6)
1949 + (c) "0.0.0.0" or "::" as a network address
1950 + (d) are too small for being publicly announced (we have decided not to
1951 + process them at the moment, as they significantly enlarge our
1952 + database without providing very helpful additional information)
1954 + This unfortunately is necessary due to brain-dead clutter across
1955 + various RIR databases, causing mismatches and eventually disruptions.
1957 + We will return False in case a network is not suitable for adding
1958 + it to our database, and True otherwise.
1961 + if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
1964 + if not network.is_global:
1965 + logging.warning("Skipping non-globally routable network: %s" % network)
1968 + if network.version == 4:
1969 + if network.prefixlen < 7:
1970 + logging.warning("Skipping too big IP chunk: %s" % network)
1973 + if network.prefixlen > 24:
1974 + logging.info("Skipping network too small to be publicly announced: %s" % network)
1977 + if str(network.network_address) == "0.0.0.0":
1978 + logging.warning("Skipping network based on 0.0.0.0: %s" % network)
1981 + elif network.version == 6:
1982 + if network.prefixlen < 10:
1983 + logging.warning("Skipping too big IP chunk: %s" % network)
1986 + if network.prefixlen > 48:
1987 + logging.info("Skipping network too small to be publicly announced: %s" % network)
1990 + if str(network.network_address) == "::":
1991 + logging.warning("Skipping network based on '::': %s" % network)
1995 + # This should not happen...
1996 + logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
1999 + # In case we have made it here, the network is considered to
2000 + # be suitable for libloc consumption...
2003 def _parse_block(self, block):
2004 # Get first line to find out what type of block this is
2006 @@ -559,22 +622,7 @@ class CLI(object):
2008 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
2010 - # Bail out in case we have processed a network covering the entire IP range, which
2011 - # is necessary to work around faulty (?) IPv6 network processing
2012 - if network.prefixlen == 0:
2013 - logging.warning("Skipping network covering the entire IP adress range: %s" % network)
2016 - # Bail out in case we have processed a network whose prefix length indicates it is
2017 - # not globally routable (we have decided not to process them at the moment, as they
2018 - # significantly enlarge our database without providing very helpful additional information)
2019 - if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
2020 - logging.info("Skipping network too small to be publicly announced: %s" % network)
2023 - # Bail out in case we have processed a non-public IP network
2024 - if network.is_private:
2025 - logging.warning("Skipping non-globally routable network: %s" % network)
2026 + if not self._check_parsed_network(network):
2029 self.db.execute("INSERT INTO _rirdata(network, country) \
2030 @@ -658,6 +706,9 @@ class CLI(object):
2031 log.warning("Invalid IP address: %s" % address)
2034 + if not self._check_parsed_network(network):
2037 self.db.execute("INSERT INTO networks(network, country) \
2038 VALUES(%s, %s) ON CONFLICT (network) DO \
2039 UPDATE SET country = excluded.country",
2043 From eee65490a10e0fe89b3834b8be176fc900084fa0 Mon Sep 17 00:00:00 2001
2044 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2045 Date: Wed, 21 Oct 2020 14:47:40 +0000
2046 Subject: [PATCH 24/70] importer.py: fetch LACNIC data via HTTPS
2048 Content-Type: text/plain; charset=UTF-8
2049 Content-Transfer-Encoding: 8bit
2051 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2052 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2054 src/python/importer.py | 2 +-
2055 1 file changed, 1 insertion(+), 1 deletion(-)
2057 diff --git a/src/python/importer.py b/src/python/importer.py
2058 index f19db4b..5f46bc3 100644
2059 --- a/src/python/importer.py
2060 +++ b/src/python/importer.py
2061 @@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
2062 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
2064 # Latin America and Caribbean Network Information Centre
2065 - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2066 + "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2068 # Réseaux IP Européens
2069 #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
2073 From 84187ab5436eb158529d6f5e2a38890b4af3ddb4 Mon Sep 17 00:00:00 2001
2074 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2075 Date: Wed, 21 Oct 2020 14:47:41 +0000
2076 Subject: [PATCH 25/70] location-importer.in: omit historic/orphaned RIR data
2078 Content-Type: text/plain; charset=UTF-8
2079 Content-Transfer-Encoding: 8bit
2081 Some RIRs include detailled information regarding networks not managed
2082 by or allocated to themselves, particually APNIC. We need to filter
2083 those networks (they usually have a characteristic network name) in
2084 order to prevent operational quirks or returning wrong country codes.
2087 Partially fixes: #12499
2089 Cc: Michael Tremer <michael.tremer@ipfire.org>
2090 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2091 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2093 src/python/location-importer.in | 38 +++++++++++++++++++++------------
2094 1 file changed, 24 insertions(+), 14 deletions(-)
2096 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2097 index f24d357..a869256 100644
2098 --- a/src/python/location-importer.in
2099 +++ b/src/python/location-importer.in
2100 @@ -494,38 +494,38 @@ class CLI(object):
2103 if not network.is_global:
2104 - logging.warning("Skipping non-globally routable network: %s" % network)
2105 + log.warning("Skipping non-globally routable network: %s" % network)
2108 if network.version == 4:
2109 if network.prefixlen < 7:
2110 - logging.warning("Skipping too big IP chunk: %s" % network)
2111 + log.warning("Skipping too big IP chunk: %s" % network)
2114 if network.prefixlen > 24:
2115 - logging.info("Skipping network too small to be publicly announced: %s" % network)
2116 + log.info("Skipping network too small to be publicly announced: %s" % network)
2119 if str(network.network_address) == "0.0.0.0":
2120 - logging.warning("Skipping network based on 0.0.0.0: %s" % network)
2121 + log.warning("Skipping network based on 0.0.0.0: %s" % network)
2124 elif network.version == 6:
2125 if network.prefixlen < 10:
2126 - logging.warning("Skipping too big IP chunk: %s" % network)
2127 + log.warning("Skipping too big IP chunk: %s" % network)
2130 if network.prefixlen > 48:
2131 - logging.info("Skipping network too small to be publicly announced: %s" % network)
2132 + log.info("Skipping network too small to be publicly announced: %s" % network)
2135 if str(network.network_address) == "::":
2136 - logging.warning("Skipping network based on '::': %s" % network)
2137 + log.warning("Skipping network based on '::': %s" % network)
2141 # This should not happen...
2142 - logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
2143 + log.warning("Skipping network of unknown family, this should not happen: %s" % network)
2146 # In case we have made it here, the network is considered to
2147 @@ -574,15 +574,22 @@ class CLI(object):
2150 def _parse_inetnum_block(self, block):
2151 - logging.debug("Parsing inetnum block:")
2152 + log.debug("Parsing inetnum block:")
2156 - logging.debug(line)
2160 key, val = split_line(line)
2162 + # Filter any inetnum records which are only referring to IP space
2163 + # not managed by that specific RIR...
2164 + if key == "netname":
2165 + if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
2166 + log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
2169 if key == "inetnum":
2170 start_address, delim, end_address = val.partition("-")
2172 @@ -594,7 +601,7 @@ class CLI(object):
2173 start_address = ipaddress.ip_address(start_address)
2174 end_address = ipaddress.ip_address(end_address)
2176 - logging.warning("Could not parse line: %s" % line)
2177 + log.warning("Could not parse line: %s" % line)
2180 # Set prefix to default
2181 @@ -611,15 +618,18 @@ class CLI(object):
2184 elif key == "country":
2185 - if val == "UNITED STATES":
2188 inetnum[key] = val.upper()
2190 # Skip empty objects
2191 if not inetnum or not "country" in inetnum:
2194 + # Skip objects with bogus country code 'ZZ'
2195 + if inetnum.get("country") == "ZZ":
2196 + log.warning("Skipping network with bogus country 'ZZ': %s" % \
2197 + (inetnum.get("inet6num") or inetnum.get("inetnum")))
2200 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
2202 if not self._check_parsed_network(network):
2206 From ebb087cfa30ec5ca0c96dcce66a91245c1ffc271 Mon Sep 17 00:00:00 2001
2207 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2208 Date: Wed, 21 Oct 2020 14:47:43 +0000
2209 Subject: [PATCH 26/70] location-importer.in: avoid log spam for too small
2212 Content-Type: text/plain; charset=UTF-8
2213 Content-Transfer-Encoding: 8bit
2215 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2216 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2218 src/python/location-importer.in | 4 ++--
2219 1 file changed, 2 insertions(+), 2 deletions(-)
2221 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2222 index a869256..864eab1 100644
2223 --- a/src/python/location-importer.in
2224 +++ b/src/python/location-importer.in
2225 @@ -503,7 +503,7 @@ class CLI(object):
2228 if network.prefixlen > 24:
2229 - log.info("Skipping network too small to be publicly announced: %s" % network)
2230 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2233 if str(network.network_address) == "0.0.0.0":
2234 @@ -516,7 +516,7 @@ class CLI(object):
2237 if network.prefixlen > 48:
2238 - log.info("Skipping network too small to be publicly announced: %s" % network)
2239 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2242 if str(network.network_address) == "::":
2246 From bbed1fd2330e8efa6b413dc152a1a6ef2d771aac Mon Sep 17 00:00:00 2001
2247 From: Michael Tremer <michael.tremer@ipfire.org>
2248 Date: Tue, 27 Oct 2020 17:14:30 +0000
2249 Subject: [PATCH 27/70] export: Flatten the tree before exporting it
2251 This patch removes the possibility that any IP address ranges
2252 might show up in multiple exported files.
2254 If this was content from the database:
2259 Then the IP address 10.0.1.1 would match for both countries.
2261 The algorithm will now break the larger /16 subnet apart into
2262 smaller subnets so that 10.0.1.0/24 is no longer overlapped.
2264 There was some time spent on this to make this as efficient
2267 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2269 src/python/export.py | 154 ++++++++++++++++++++++++++++++-------------
2270 1 file changed, 110 insertions(+), 44 deletions(-)
2272 diff --git a/src/python/export.py b/src/python/export.py
2273 index 5eaf43f..dd44332 100644
2274 --- a/src/python/export.py
2275 +++ b/src/python/export.py
2276 @@ -89,28 +89,55 @@ class OutputWriter(object):
2278 def write(self, network, subnets):
2279 if self.flatten and self._flatten(network):
2280 - log.debug("Skipping writing network %s" % network)
2281 + log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
2284 + # Convert network into a Python object
2285 + _network = ipaddress.ip_network("%s" % network)
2287 # Write the network when it has no subnets
2289 - network = ipaddress.ip_network("%s" % network)
2290 - return self._write_network(network)
2291 + log.debug("Writing %s to %s" % (_network, self.f))
2292 + return self._write_network(_network)
2294 + # Convert subnets into Python objects
2295 + _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
2297 + # Split the network into smaller bits so that
2298 + # we can accomodate for any gaps in it later
2300 + for _subnet in _subnets:
2302 + _network.address_exclude(_subnet)
2305 + # Clear the list of all subnets
2308 + # Check if all subnets to not overlap with anything else
2310 + subnet_to_check = to_check.pop()
2312 - # Collect all matching subnets
2313 - matching_subnets = []
2314 + for _subnet in _subnets:
2315 + # Drop this subnet if it equals one of the subnets
2316 + # or if it is subnet of one of them
2317 + if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
2320 - for subnet in sorted(subnets):
2321 - # Try to find the subnet in the database
2322 - n = self.db.lookup("%s" % subnet.network_address)
2323 + # Break it down if it overlaps
2324 + if subnet_to_check.overlaps(_subnet):
2326 + subnet_to_check.address_exclude(_subnet)
2330 - # No entry or matching country means those IP addresses belong here
2331 - if not n or n.country_code == network.country_code:
2332 - matching_subnets.append(subnet)
2333 + # Add the subnet again as it passed the check
2335 + subnets.append(subnet_to_check)
2337 # Write all networks as compact as possible
2338 - for network in ipaddress.collapse_addresses(matching_subnets):
2339 - log.debug("Writing %s to database" % network)
2340 + for network in ipaddress.collapse_addresses(subnets):
2341 + log.debug("Writing %s to %s" % (network, self.f))
2342 self._write_network(network)
2345 @@ -206,40 +233,40 @@ class Exporter(object):
2346 # Get all networks that match the family
2347 networks = self.db.search_networks(family=family)
2349 - # Materialise the generator into a list (uses quite some memory)
2350 - networks = list(networks)
2351 + # Create a stack with all networks in order where we can put items back
2352 + # again and retrieve them in the next iteration.
2353 + networks = BufferedStack(networks)
2355 # Walk through all networks
2356 - for i, network in enumerate(networks):
2357 - _network = ipaddress.ip_network("%s" % network)
2359 - # Search for all subnets
2362 - while i < len(networks):
2363 - subnet = networks[i+1]
2365 - if subnet.is_subnet_of(network):
2366 - _subnet = ipaddress.ip_network("%s" % subnet)
2368 - subnets.add(_subnet)
2369 - subnets.update(_network.address_exclude(_subnet))
2373 + for network in networks:
2374 + # Collect all networks which are a subnet of network
2376 + for subnet in networks:
2377 + # If the next subnet was not a subnet, we have to push
2378 + # it back on the stack and break this loop
2379 + if not subnet.is_subnet_of(network):
2380 + networks.push(subnet)
2383 + subnets.append(subnet)
2385 # Write matching countries
2387 - writers[network.country_code].write(network, subnets)
2390 + if network.country_code and network.country_code in writers:
2391 + # Mismatching subnets
2393 + subnet for subnet in subnets if not network.country_code == subnet.country_code
2396 + writers[network.country_code].write(network, gaps)
2398 # Write matching ASNs
2400 - writers[network.asn].write(network, subnets)
2403 + if network.asn and network.asn in writers:
2404 + # Mismatching subnets
2406 + subnet for subnet in subnets if not network.asn == subnet.asn
2409 + writers[network.asn].write(network, gaps)
2413 @@ -247,10 +274,19 @@ class Exporter(object):
2414 # Fetch the "fake" country code
2415 country = flags[flag]
2418 - writers[country].write(network, subnets)
2421 + if not country in writers:
2425 + subnet for subnet in subnets
2426 + if not subnet.has_flag(flag)
2429 + writers[country].write(network, gaps)
2431 + # Push all subnets back onto the stack
2432 + for subnet in reversed(subnets):
2433 + networks.push(subnet)
2435 # Write everything to the filesystem
2436 for writer in writers.values():
2437 @@ -262,3 +298,33 @@ class Exporter(object):
2440 return os.path.join(directory, filename)
2443 +class BufferedStack(object):
2445 + This class takes an iterator and when being iterated
2446 + over it returns objects from that iterator for as long
2449 + It additionally has a function to put an item back on
2450 + the back so that it will be returned again at the next
2453 + def __init__(self, iterator):
2454 + self.iterator = iterator
2457 + def __iter__(self):
2460 + def __next__(self):
2462 + return self.stack.pop(0)
2464 + return next(self.iterator)
2466 + def push(self, elem):
2468 + Takes an element and puts it on the stack
2470 + self.stack.insert(0, elem)
2474 From e99a72265c1ba2194b61663eda7e9f14e0083016 Mon Sep 17 00:00:00 2001
2475 From: Michael Tremer <michael.tremer@ipfire.org>
2476 Date: Wed, 28 Oct 2020 09:52:36 +0000
2477 Subject: [PATCH 28/70] location: Fix Python syntax error in verify()
2479 The database is now being opened before the requested
2480 method is called and handle_verify() wasn't updated.
2482 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2484 src/python/location.in | 8 +-------
2485 1 file changed, 1 insertion(+), 7 deletions(-)
2487 diff --git a/src/python/location.in b/src/python/location.in
2488 index 44ad726..b5e5758 100644
2489 --- a/src/python/location.in
2490 +++ b/src/python/location.in
2491 @@ -453,13 +453,7 @@ class CLI(object):
2495 - def handle_verify(self, ns):
2497 - db = location.Database(ns.database)
2498 - except FileNotFoundError as e:
2499 - log.error("%s: %s" % (ns.database, e))
2502 + def handle_verify(self, db, ns):
2503 # Verify the database
2504 with open(ns.public_key, "r") as f:
2505 if not db.verify(f):
2509 From 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 Mon Sep 17 00:00:00 2001
2510 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2511 Date: Thu, 29 Oct 2020 07:25:53 -0700
2512 Subject: [PATCH 29/70] location update: Remove double conversion of timestamps
2514 Content-Type: text/plain; charset=UTF-8
2515 Content-Transfer-Encoding: 8bit
2517 This caused that the timestamp in DNS was misinterpreted
2518 as local time and often, databases could not be downloaded.
2520 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2522 src/python/location.in | 5 +----
2523 1 file changed, 1 insertion(+), 4 deletions(-)
2525 diff --git a/src/python/location.in b/src/python/location.in
2526 index b5e5758..070640c 100644
2527 --- a/src/python/location.in
2528 +++ b/src/python/location.in
2529 @@ -421,11 +421,8 @@ class CLI(object):
2530 # Fetch the timestamp we need from DNS
2531 t = location.discover_latest_version()
2533 - # Parse timestamp into datetime format
2534 - timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
2536 # Check the version of the local database
2537 - if db and timestamp and db.created_at >= timestamp.timestamp():
2538 + if db and t and db.created_at >= t:
2539 log.info("Already on the latest version")
2545 From 60c1ac0307312614bd6980d30b44bb59b5a6ab6e Mon Sep 17 00:00:00 2001
2546 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2547 Date: Thu, 29 Oct 2020 07:36:46 -0700
2548 Subject: [PATCH 30/70] location.in: do not confuse UTC with local time zones
2550 Content-Type: text/plain; charset=UTF-8
2551 Content-Transfer-Encoding: 8bit
2553 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2555 src/python/location.in | 12 +++++-------
2556 1 file changed, 5 insertions(+), 7 deletions(-)
2558 diff --git a/src/python/location.in b/src/python/location.in
2559 index 070640c..0d09210 100644
2560 --- a/src/python/location.in
2561 +++ b/src/python/location.in
2562 @@ -398,10 +398,7 @@ class CLI(object):
2564 def handle_update(self, db, ns):
2566 - now = datetime.datetime.utcnow()
2568 - # Parse the database timestamp
2569 - t = datetime.datetime.utcfromtimestamp(db.created_at)
2572 if ns.cron == "daily":
2573 delta = datetime.timedelta(days=1)
2574 @@ -410,11 +407,12 @@ class CLI(object):
2575 elif ns.cron == "monthly":
2576 delta = datetime.timedelta(days=30)
2578 + delta = delta.total_seconds()
2580 # Check if the database has recently been updated
2581 - if t >= (now - delta):
2582 + if db.created_at >= (now - delta):
2584 - _("The database has been updated recently (%s)") % \
2585 - format_timedelta(now - t),
2586 + _("The database has been updated recently"),
2593 From e7d612e5219ef9ba612ed404e4e2c174110d3dd7 Mon Sep 17 00:00:00 2001
2594 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2595 Date: Tue, 3 Nov 2020 15:31:08 +0000
2596 Subject: [PATCH 31/70] location-importer.in: always convert organisation
2597 handles into upper cases
2599 Content-Type: text/plain; charset=UTF-8
2600 Content-Transfer-Encoding: 8bit
2604 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2605 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2607 src/python/location-importer.in | 6 ++++--
2608 1 file changed, 4 insertions(+), 2 deletions(-)
2610 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2611 index 864eab1..2dec89e 100644
2612 --- a/src/python/location-importer.in
2613 +++ b/src/python/location-importer.in
2614 @@ -560,7 +560,7 @@ class CLI(object):
2615 autnum["asn"] = m.group(2)
2619 + autnum[key] = val.upper()
2621 # Skip empty objects
2623 @@ -646,7 +646,9 @@ class CLI(object):
2625 key, val = split_line(line)
2627 - if key in ("organisation", "org-name"):
2628 + if key == "organisation":
2629 + org[key] = val.upper()
2630 + elif key == "org-name":
2633 # Skip empty objects
2637 From e96704f43acca1a8f56d9a680cce281f5e587ec5 Mon Sep 17 00:00:00 2001
2638 From: Michael Tremer <michael.tremer@ipfire.org>
2639 Date: Wed, 11 Nov 2020 21:16:45 +0000
2640 Subject: [PATCH 32/70] test: Add tests for database enumerator
2642 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2644 src/test-database.c | 53 +++++++++++++++++++++++++++++++++++++++++++++
2645 1 file changed, 53 insertions(+)
2647 diff --git a/src/test-database.c b/src/test-database.c
2648 index b4a75c4..4aef94e 100644
2649 --- a/src/test-database.c
2650 +++ b/src/test-database.c
2651 @@ -38,6 +38,14 @@ const char* DESCRIPTION =
2652 "Maecenas ut venenatis nunc.";
2653 const char* LICENSE = "CC";
2655 +const char* networks[] = {
2657 + "2001:db8:1000::/48",
2658 + "2001:db8:2000::/48",
2659 + "2001:db8:2020::/48",
2663 static int attempt_to_open(struct loc_ctx* ctx, char* path) {
2664 FILE* f = fopen(path, "r");
2666 @@ -139,6 +147,24 @@ int main(int argc, char** argv) {
2670 + struct loc_network* network = NULL;
2672 + // Add some networks
2673 + const char** n = networks;
2675 + err = loc_writer_add_network(writer, &network, *n);
2677 + fprintf(stderr, "Could not add network %s\n", *n);
2678 + exit(EXIT_FAILURE);
2682 + loc_network_set_country_code(network, "XX");
2688 FILE* f = tmpfile();
2690 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
2691 @@ -170,6 +196,33 @@ int main(int argc, char** argv) {
2696 + struct loc_database_enumerator* enumerator;
2697 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
2699 + fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
2700 + exit(EXIT_FAILURE);
2703 + // Walk through all networks
2705 + err = loc_database_enumerator_next_network(enumerator, &network);
2707 + fprintf(stderr, "Error fetching the next network: %d\n", err);
2708 + exit(EXIT_FAILURE);
2714 + char* s = loc_network_str(network);
2715 + printf("Got network: %s\n", s);
2719 + // Free the enumerator
2720 + loc_database_enumerator_unref(enumerator);
2722 // Close the database
2723 loc_database_unref(db);
2728 From ecce288da39a2c0eb60076050ca21e9619f61844 Mon Sep 17 00:00:00 2001
2729 From: Michael Tremer <michael.tremer@ipfire.org>
2730 Date: Wed, 11 Nov 2020 23:01:19 +0000
2731 Subject: [PATCH 33/70] networks: Add list to manage groups of networks
2733 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2735 src/libloc.sym | 11 ++++++
2736 src/loc/network.h | 12 ++++++
2737 src/network.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++
2738 3 files changed, 119 insertions(+)
2740 diff --git a/src/libloc.sym b/src/libloc.sym
2741 index b8296eb..6ef2d27 100644
2742 --- a/src/libloc.sym
2743 +++ b/src/libloc.sym
2744 @@ -98,6 +98,17 @@ global:
2749 + loc_network_list_clear;
2750 + loc_network_list_empty;
2751 + loc_network_list_get;
2752 + loc_network_list_new;
2753 + loc_network_list_pop;
2754 + loc_network_list_push;
2755 + loc_network_list_ref;
2756 + loc_network_list_size;
2757 + loc_network_list_unref;
2761 loc_writer_add_country;
2762 diff --git a/src/loc/network.h b/src/loc/network.h
2763 index 70c3803..fd20812 100644
2764 --- a/src/loc/network.h
2765 +++ b/src/loc/network.h
2766 @@ -56,6 +56,18 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
2768 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
2771 +struct loc_network_list;
2772 +int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
2773 +struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
2774 +struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
2775 +size_t loc_network_list_size(struct loc_network_list* list);
2776 +int loc_network_list_empty(struct loc_network_list* list);
2777 +void loc_network_list_clear(struct loc_network_list* list);
2778 +struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
2779 +int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
2780 +struct loc_network* loc_network_list_pop(struct loc_network_list* list);
2782 #ifdef LIBLOC_PRIVATE
2784 int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
2785 diff --git a/src/network.c b/src/network.c
2786 index d7b1645..c9e7979 100644
2789 @@ -746,3 +746,99 @@ LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node)
2790 LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2791 return loc_network_ref(node->network);
2796 +struct loc_network_list {
2797 + struct loc_ctx* ctx;
2800 + struct loc_network* list[1024];
2805 +LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
2806 + struct loc_network_list** list) {
2807 + struct loc_network_list* l = calloc(1, sizeof(*l));
2811 + l->ctx = loc_ref(ctx);
2814 + // Do not allow this list to grow larger than this
2815 + l->max_size = 1024;
2817 + DEBUG(l->ctx, "Network list allocated at %p\n", l);
2822 +LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
2828 +static void loc_network_list_free(struct loc_network_list* list) {
2829 + DEBUG(list->ctx, "Releasing network list at %p\n", list);
2831 + for (unsigned int i = 0; i < list->size; i++)
2832 + loc_network_unref(list->list[i]);
2834 + loc_unref(list->ctx);
2838 +LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
2842 + if (--list->refcount > 0)
2845 + loc_network_list_free(list);
2849 +LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
2850 + return list->size;
2853 +LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
2854 + return list->size == 0;
2857 +LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
2858 + for (unsigned int i = 0; i < list->size; i++)
2859 + loc_network_unref(list->list[i]);
2864 +LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
2866 + if (index >= list->size)
2869 + return loc_network_ref(list->list[index]);
2872 +LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
2873 + // Check if we have space left
2874 + if (list->size == list->max_size)
2877 + list->list[list->size++] = loc_network_ref(network);
2882 +LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
2883 + // Return nothing when empty
2884 + if (loc_network_list_empty(list))
2887 + return list->list[list->size--];
2892 From 8b2205272b7872a1458ad87811abf58609f38ad4 Mon Sep 17 00:00:00 2001
2893 From: Michael Tremer <michael.tremer@ipfire.org>
2894 Date: Thu, 12 Nov 2020 13:57:35 +0000
2895 Subject: [PATCH 34/70] networks: Add function to dump lists
2897 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2899 src/libloc.sym | 1 +
2900 src/loc/network.h | 1 +
2901 src/network.c | 14 ++++++++++++++
2902 3 files changed, 16 insertions(+)
2904 diff --git a/src/libloc.sym b/src/libloc.sym
2905 index 6ef2d27..a5641c6 100644
2906 --- a/src/libloc.sym
2907 +++ b/src/libloc.sym
2908 @@ -100,6 +100,7 @@ global:
2911 loc_network_list_clear;
2912 + loc_network_list_dump;
2913 loc_network_list_empty;
2914 loc_network_list_get;
2915 loc_network_list_new;
2916 diff --git a/src/loc/network.h b/src/loc/network.h
2917 index fd20812..44c50a4 100644
2918 --- a/src/loc/network.h
2919 +++ b/src/loc/network.h
2920 @@ -64,6 +64,7 @@ struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
2921 size_t loc_network_list_size(struct loc_network_list* list);
2922 int loc_network_list_empty(struct loc_network_list* list);
2923 void loc_network_list_clear(struct loc_network_list* list);
2924 +void loc_network_list_dump(struct loc_network_list* list);
2925 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
2926 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
2927 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
2928 diff --git a/src/network.c b/src/network.c
2929 index c9e7979..0977406 100644
2932 @@ -817,6 +817,20 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
2936 +LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
2937 + struct loc_network* network;
2940 + for (unsigned int i = 0; i < list->size; i++) {
2941 + network = list->list[i];
2943 + s = loc_network_str(network);
2945 + INFO(list->ctx, "%s\n", s);
2950 LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
2952 if (index >= list->size)
2956 From 850e75167e8e03fe8b951992c9f7bc2ccb9fb711 Mon Sep 17 00:00:00 2001
2957 From: Michael Tremer <michael.tremer@ipfire.org>
2958 Date: Thu, 12 Nov 2020 14:18:40 +0000
2959 Subject: [PATCH 35/70] network: Add functions to break network into subnets
2961 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2963 src/libloc.sym | 5 +
2964 src/loc/network.h | 6 +
2965 src/network.c | 290 ++++++++++++++++++++++++++++++++++++++++++++-
2966 src/test-network.c | 61 ++++++++++
2967 4 files changed, 361 insertions(+), 1 deletion(-)
2969 diff --git a/src/libloc.sym b/src/libloc.sym
2970 index a5641c6..fcb9ea5 100644
2971 --- a/src/libloc.sym
2972 +++ b/src/libloc.sym
2973 @@ -80,6 +80,8 @@ global:
2976 loc_network_address_family;
2978 + loc_network_exclude;
2979 loc_network_format_first_address;
2980 loc_network_format_last_address;
2981 loc_network_get_asn;
2982 @@ -96,6 +98,7 @@ global:
2983 loc_network_set_country_code;
2984 loc_network_set_flag;
2986 + loc_network_subnets;
2990 @@ -107,7 +110,9 @@ global:
2991 loc_network_list_pop;
2992 loc_network_list_push;
2993 loc_network_list_ref;
2994 + loc_network_list_reverse;
2995 loc_network_list_size;
2996 + loc_network_list_sort;
2997 loc_network_list_unref;
3000 diff --git a/src/loc/network.h b/src/loc/network.h
3001 index 44c50a4..4e51a62 100644
3002 --- a/src/loc/network.h
3003 +++ b/src/loc/network.h
3004 @@ -54,7 +54,11 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
3005 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
3006 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
3008 +int loc_network_eq(struct loc_network* self, struct loc_network* other);
3009 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
3010 +struct loc_network_list* loc_network_subnets(struct loc_network* network);
3011 +struct loc_network_list* loc_network_exclude(
3012 + struct loc_network* self, struct loc_network* other);
3015 struct loc_network_list;
3016 @@ -68,6 +72,8 @@ void loc_network_list_dump(struct loc_network_list* list);
3017 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
3018 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
3019 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
3020 +void loc_network_list_sort(struct loc_network_list* list);
3021 +void loc_network_list_reverse(struct loc_network_list* list);
3023 #ifdef LIBLOC_PRIVATE
3025 diff --git a/src/network.c b/src/network.c
3026 index 0977406..6c08070 100644
3029 @@ -97,6 +97,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
3033 +static struct in6_addr address_increment(const struct in6_addr* address) {
3034 + struct in6_addr a = *address;
3036 + for (int octet = 15; octet >= 0; octet--) {
3037 + if (a.s6_addr[octet] < 255) {
3038 + a.s6_addr[octet]++;
3041 + a.s6_addr[octet] = 0;
3048 LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
3049 struct in6_addr* address, unsigned int prefix) {
3050 // Address cannot be unspecified
3051 @@ -405,6 +420,69 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
3052 return loc_network_has_flag(network, flag);
3055 +LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
3056 +#ifdef ENABLE_DEBUG
3057 + char* n1 = loc_network_str(self);
3058 + char* n2 = loc_network_str(other);
3060 + DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
3066 + // Family must be the same
3067 + if (self->family != other->family) {
3068 + DEBUG(self->ctx, " Family mismatch\n");
3073 + // The start address must be the same
3074 + if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
3075 + DEBUG(self->ctx, " Address mismatch\n");
3080 + // The prefix length must be the same
3081 + if (self->prefix != other->prefix) {
3082 + DEBUG(self->ctx, " Prefix mismatch\n");
3086 + DEBUG(self->ctx, " Yes!\n");
3091 +static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
3092 + // Families must match
3093 + if (self->family != other->family)
3096 + int r = in6_addr_cmp(&self->first_address, &other->first_address);
3111 + if (self->prefix > other->prefix)
3118 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
3119 // If the start address of the other network is smaller than this network,
3120 // it cannot be a subnet.
3121 @@ -419,6 +497,175 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net
3125 +LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
3126 + struct loc_network_list* list;
3128 + // New prefix length
3129 + unsigned int prefix = network->prefix + 1;
3131 + // Check if the new prefix is valid
3132 + if (valid_prefix(&network->first_address, prefix))
3135 + // Create a new list with the result
3136 + int r = loc_network_list_new(network->ctx, &list);
3138 + ERROR(network->ctx, "Could not create network list: %d\n", r);
3142 + struct loc_network* subnet1 = NULL;
3143 + struct loc_network* subnet2 = NULL;
3145 + // Create the first half of the network
3146 + r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix);
3150 + // The next subnet starts after the first one
3151 + struct in6_addr first_address = address_increment(&subnet1->last_address);
3153 + // Create the second half of the network
3154 + r = loc_network_new(network->ctx, &subnet2, &first_address, prefix);
3158 + // Push the both onto the stack (in reverse order)
3159 + r = loc_network_list_push(list, subnet2);
3163 + r = loc_network_list_push(list, subnet1);
3167 + loc_network_unref(subnet1);
3168 + loc_network_unref(subnet2);
3174 + loc_network_unref(subnet1);
3177 + loc_network_unref(subnet2);
3180 + loc_network_list_unref(list);
3185 +LOC_EXPORT struct loc_network_list* loc_network_exclude(
3186 + struct loc_network* self, struct loc_network* other) {
3187 + struct loc_network_list* list;
3189 +#ifdef ENABLE_DEBUG
3190 + char* n1 = loc_network_str(self);
3191 + char* n2 = loc_network_str(other);
3193 + DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
3199 + // Family must match
3200 + if (self->family != other->family) {
3201 + DEBUG(self->ctx, "Family mismatch\n");
3206 + // Other must be a subnet of self
3207 + if (!loc_network_is_subnet_of(other, self)) {
3208 + DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
3213 + // We cannot perform this operation if both networks equal
3214 + if (loc_network_eq(self, other)) {
3215 + DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
3220 + // Create a new list with the result
3221 + int r = loc_network_list_new(self->ctx, &list);
3223 + ERROR(self->ctx, "Could not create network list: %d\n", r);
3227 + struct loc_network_list* subnets = loc_network_subnets(self);
3229 + struct loc_network* subnet1 = NULL;
3230 + struct loc_network* subnet2 = NULL;
3233 + // Fetch both subnets
3234 + subnet1 = loc_network_list_get(subnets, 0);
3235 + subnet2 = loc_network_list_get(subnets, 1);
3238 + loc_network_list_unref(subnets);
3241 + if (loc_network_eq(other, subnet1)) {
3242 + r = loc_network_list_push(list, subnet2);
3246 + } else if (loc_network_eq(other, subnet2)) {
3247 + r = loc_network_list_push(list, subnet1);
3251 + } else if (loc_network_is_subnet_of(other, subnet1)) {
3252 + r = loc_network_list_push(list, subnet2);
3256 + subnets = loc_network_subnets(subnet1);
3258 + } else if (loc_network_is_subnet_of(other, subnet2)) {
3259 + r = loc_network_list_push(list, subnet1);
3263 + subnets = loc_network_subnets(subnet2);
3266 + ERROR(self->ctx, "We should never get here\n");
3270 + loc_network_unref(subnet1);
3271 + loc_network_unref(subnet2);
3274 +#ifdef ENABLE_DEBUG
3275 + loc_network_list_dump(list);
3278 + // Return the result
3283 + loc_network_unref(subnet1);
3286 + loc_network_unref(subnet2);
3289 + loc_network_list_unref(list);
3294 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
3296 loc_country_code_copy(dbobj->country_code, network->country_code);
3297 @@ -854,5 +1101,46 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
3298 if (loc_network_list_empty(list))
3301 - return list->list[list->size--];
3302 + return list->list[--list->size];
3305 +static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
3306 + // Do nothing for invalid indices
3307 + if (i1 >= list->size || i2 >= list->size)
3310 + DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
3312 + struct loc_network* network1 = list->list[i1];
3313 + struct loc_network* network2 = list->list[i2];
3315 + list->list[i1] = network2;
3316 + list->list[i2] = network1;
3319 +LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
3320 + unsigned int i = 0;
3321 + unsigned int j = list->size - 1;
3324 + loc_network_list_swap(list, i++, j--);
3328 +LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
3329 + unsigned int n = list->size;
3335 + for (unsigned int i = 1; i < n; i++) {
3336 + if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
3337 + loc_network_list_swap(list, i-1, i);
3343 + } while (swapped);
3345 diff --git a/src/test-network.c b/src/test-network.c
3346 index b6776b4..af1b2e6 100644
3347 --- a/src/test-network.c
3348 +++ b/src/test-network.c
3349 @@ -118,6 +118,19 @@ int main(int argc, char** argv) {
3350 size_t nodes = loc_network_tree_count_nodes(tree);
3351 printf("The tree has %zu nodes\n", nodes);
3353 + // Check equals function
3354 + err = loc_network_eq(network1, network1);
3356 + fprintf(stderr, "Network is not equal with itself\n");
3357 + exit(EXIT_FAILURE);
3360 + err = loc_network_eq(network1, network2);
3362 + fprintf(stderr, "Networks equal unexpectedly\n");
3363 + exit(EXIT_FAILURE);
3366 // Check subnet function
3367 err = loc_network_is_subnet_of(network1, network2);
3369 @@ -131,6 +144,54 @@ int main(int argc, char** argv) {
3373 + // Make list of subnets
3374 + struct loc_network_list* subnets = loc_network_subnets(network1);
3376 + fprintf(stderr, "Could not find subnets of network\n");
3377 + exit(EXIT_FAILURE);
3380 + loc_network_list_dump(subnets);
3382 + while (!loc_network_list_empty(subnets)) {
3383 + struct loc_network* subnet = loc_network_list_pop(subnets);
3385 + fprintf(stderr, "Received an empty subnet\n");
3386 + exit(EXIT_FAILURE);
3389 + char* s = loc_network_str(subnet);
3390 + printf("Received subnet %s\n", s);
3393 + if (!loc_network_is_subnet_of(subnet, network1)) {
3394 + fprintf(stderr, "Not a subnet\n");
3395 + exit(EXIT_FAILURE);
3398 + loc_network_unref(subnet);
3401 + loc_network_list_unref(subnets);
3403 + struct loc_network_list* excluded = loc_network_exclude(network1, network2);
3405 + fprintf(stderr, "Could not create excluded list\n");
3406 + exit(EXIT_FAILURE);
3409 + loc_network_list_dump(excluded);
3412 + loc_network_list_reverse(excluded);
3413 + loc_network_list_dump(excluded);
3415 + // Sort them and dump them again
3416 + loc_network_list_sort(excluded);
3417 + loc_network_list_dump(excluded);
3419 + loc_network_list_unref(excluded);
3421 // Create a database
3422 struct loc_writer* writer;
3423 err = loc_writer_new(ctx, &writer, NULL, NULL);
3427 From 6159d384c4a98fe45ec52522e2950719e4982d80 Mon Sep 17 00:00:00 2001
3428 From: Michael Tremer <michael.tremer@ipfire.org>
3429 Date: Thu, 12 Nov 2020 14:24:58 +0000
3430 Subject: [PATCH 36/70] networks: Add function to check if two networks overlap
3432 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3434 src/libloc.sym | 1 +
3435 src/loc/network.h | 1 +
3436 src/network.c | 16 ++++++++++++++++
3437 3 files changed, 18 insertions(+)
3439 diff --git a/src/libloc.sym b/src/libloc.sym
3440 index fcb9ea5..0c2835b 100644
3441 --- a/src/libloc.sym
3442 +++ b/src/libloc.sym
3443 @@ -93,6 +93,7 @@ global:
3444 loc_network_match_flag;
3446 loc_network_new_from_string;
3447 + loc_network_overlaps;
3449 loc_network_set_asn;
3450 loc_network_set_country_code;
3451 diff --git a/src/loc/network.h b/src/loc/network.h
3452 index 4e51a62..ef13756 100644
3453 --- a/src/loc/network.h
3454 +++ b/src/loc/network.h
3455 @@ -55,6 +55,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
3456 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
3458 int loc_network_eq(struct loc_network* self, struct loc_network* other);
3459 +int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
3460 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
3461 struct loc_network_list* loc_network_subnets(struct loc_network* network);
3462 struct loc_network_list* loc_network_exclude(
3463 diff --git a/src/network.c b/src/network.c
3464 index 6c08070..d826511 100644
3467 @@ -483,6 +483,22 @@ static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
3471 +LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
3472 + if (loc_network_match_address(self, &other->first_address) == 0)
3475 + if (loc_network_match_address(self, &other->last_address) == 0)
3478 + if (loc_network_match_address(other, &self->first_address) == 0)
3481 + if (loc_network_match_address(other, &self->last_address) == 0)
3487 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
3488 // If the start address of the other network is smaller than this network,
3489 // it cannot be a subnet.
3493 From e52ba21761f27e592040a2793b2a26bbeeeecc05 Mon Sep 17 00:00:00 2001
3494 From: Michael Tremer <michael.tremer@ipfire.org>
3495 Date: Thu, 12 Nov 2020 14:28:15 +0000
3496 Subject: [PATCH 37/70] networks: Add function to check if network is part of a
3499 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3501 src/libloc.sym | 1 +
3502 src/loc/network.h | 1 +
3503 src/network.c | 9 +++++++++
3504 3 files changed, 11 insertions(+)
3506 diff --git a/src/libloc.sym b/src/libloc.sym
3507 index 0c2835b..f1b63a2 100644
3508 --- a/src/libloc.sym
3509 +++ b/src/libloc.sym
3510 @@ -104,6 +104,7 @@ global:
3513 loc_network_list_clear;
3514 + loc_network_list_contains;
3515 loc_network_list_dump;
3516 loc_network_list_empty;
3517 loc_network_list_get;
3518 diff --git a/src/loc/network.h b/src/loc/network.h
3519 index ef13756..7804512 100644
3520 --- a/src/loc/network.h
3521 +++ b/src/loc/network.h
3522 @@ -73,6 +73,7 @@ void loc_network_list_dump(struct loc_network_list* list);
3523 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
3524 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
3525 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
3526 +int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
3527 void loc_network_list_sort(struct loc_network_list* list);
3528 void loc_network_list_reverse(struct loc_network_list* list);
3530 diff --git a/src/network.c b/src/network.c
3531 index d826511..fcbdc59 100644
3534 @@ -1120,6 +1120,15 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
3535 return list->list[--list->size];
3538 +LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
3539 + for (unsigned int i = 0; i < list->size; i++) {
3540 + if (loc_network_eq(list->list[i], network))
3547 static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
3548 // Do nothing for invalid indices
3549 if (i1 >= list->size || i2 >= list->size)
3553 From f802f3a4decf4827ecc8bcabe269ae9f94f7f32d Mon Sep 17 00:00:00 2001
3554 From: Michael Tremer <michael.tremer@ipfire.org>
3555 Date: Thu, 12 Nov 2020 14:33:22 +0000
3556 Subject: [PATCH 38/70] networks: Add function to merge two lists
3558 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3560 src/libloc.sym | 1 +
3561 src/loc/network.h | 1 +
3562 src/network.c | 13 +++++++++++++
3563 3 files changed, 15 insertions(+)
3565 diff --git a/src/libloc.sym b/src/libloc.sym
3566 index f1b63a2..c0b6b1f 100644
3567 --- a/src/libloc.sym
3568 +++ b/src/libloc.sym
3569 @@ -108,6 +108,7 @@ global:
3570 loc_network_list_dump;
3571 loc_network_list_empty;
3572 loc_network_list_get;
3573 + loc_network_list_merge;
3574 loc_network_list_new;
3575 loc_network_list_pop;
3576 loc_network_list_push;
3577 diff --git a/src/loc/network.h b/src/loc/network.h
3578 index 7804512..e30d91c 100644
3579 --- a/src/loc/network.h
3580 +++ b/src/loc/network.h
3581 @@ -76,6 +76,7 @@ struct loc_network* loc_network_list_pop(struct loc_network_list* list);
3582 int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
3583 void loc_network_list_sort(struct loc_network_list* list);
3584 void loc_network_list_reverse(struct loc_network_list* list);
3585 +int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
3587 #ifdef LIBLOC_PRIVATE
3589 diff --git a/src/network.c b/src/network.c
3590 index fcbdc59..541286d 100644
3593 @@ -1169,3 +1169,16 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
3598 +LOC_EXPORT int loc_network_list_merge(
3599 + struct loc_network_list* self, struct loc_network_list* other) {
3602 + for (unsigned int i = 0; i < other->size; i++) {
3603 + r = loc_network_list_push(self, other->list[i]);
3613 From 6d22a179dffd08fcf2a44aafb361725ab22486d4 Mon Sep 17 00:00:00 2001
3614 From: Michael Tremer <michael.tremer@ipfire.org>
3615 Date: Thu, 12 Nov 2020 14:35:43 +0000
3616 Subject: [PATCH 39/70] network: Make lists unique
3618 Networks that are in the list won't be added again
3620 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3622 src/network.c | 4 ++++
3623 1 file changed, 4 insertions(+)
3625 diff --git a/src/network.c b/src/network.c
3626 index 541286d..44571b3 100644
3629 @@ -1103,6 +1103,10 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
3632 LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
3633 + // Do not add networks that are already on the list
3634 + if (loc_network_list_contains(list, network))
3637 // Check if we have space left
3638 if (list->size == list->max_size)
3643 From 681ff05cb7cdf230d38abf09a330a31498e265a4 Mon Sep 17 00:00:00 2001
3644 From: Michael Tremer <michael.tremer@ipfire.org>
3645 Date: Thu, 12 Nov 2020 19:21:13 +0000
3646 Subject: [PATCH 40/70] database: Pass flag to enumerator to flatten output
3648 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3650 src/database.c | 8 +++++++-
3651 src/loc/database.h | 6 +++++-
3652 src/perl/Location.xs | 2 +-
3653 src/python/database.c | 25 ++++++++++++++++++-------
3654 4 files changed, 31 insertions(+), 10 deletions(-)
3656 diff --git a/src/database.c b/src/database.c
3657 index fa1dad0..9baab33 100644
3658 --- a/src/database.c
3659 +++ b/src/database.c
3660 @@ -104,6 +104,9 @@ struct loc_database_enumerator {
3661 enum loc_network_flags flags;
3664 + // Flatten output?
3667 // Index of the AS we are looking at
3668 unsigned int as_index;
3670 @@ -933,7 +936,7 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
3673 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
3674 - struct loc_database* db, enum loc_database_enumerator_mode mode) {
3675 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
3676 struct loc_database_enumerator* e = calloc(1, sizeof(*e));
3679 @@ -944,6 +947,9 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
3683 + // Flatten output?
3684 + e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
3686 // Initialise graph search
3687 //e->network_stack[++e->network_stack_depth] = 0;
3688 e->network_stack_depth = 1;
3689 diff --git a/src/loc/database.h b/src/loc/database.h
3690 index 43173dd..14eb5ea 100644
3691 --- a/src/loc/database.h
3692 +++ b/src/loc/database.h
3693 @@ -55,9 +55,13 @@ enum loc_database_enumerator_mode {
3694 LOC_DB_ENUMERATE_COUNTRIES = 3,
3697 +enum loc_database_enumerator_flags {
3698 + LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
3701 struct loc_database_enumerator;
3702 int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
3703 - struct loc_database* db, enum loc_database_enumerator_mode mode);
3704 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
3705 struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
3706 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
3708 diff --git a/src/perl/Location.xs b/src/perl/Location.xs
3709 index dcf3f0d..b7676d2 100644
3710 --- a/src/perl/Location.xs
3711 +++ b/src/perl/Location.xs
3712 @@ -125,7 +125,7 @@ database_countries(db)
3714 // Create Database enumerator
3715 struct loc_database_enumerator* enumerator;
3716 - int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
3717 + int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
3720 croak("Could not create a database enumerator\n");
3721 diff --git a/src/python/database.c b/src/python/database.c
3722 index 1013a58..7f8c2c2 100644
3723 --- a/src/python/database.c
3724 +++ b/src/python/database.c
3725 @@ -207,10 +207,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
3726 return (PyObject*)self;
3729 -static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
3730 +static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
3731 struct loc_database_enumerator* enumerator;
3733 - int r = loc_database_enumerator_new(&enumerator, self->db, what);
3734 + int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
3736 PyErr_SetFromErrno(PyExc_SystemError);
3738 @@ -223,7 +223,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
3741 static PyObject* Database_ases(DatabaseObject* self) {
3742 - return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
3743 + return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
3746 static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
3747 @@ -234,7 +234,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
3749 struct loc_database_enumerator* enumerator;
3751 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
3752 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
3754 PyErr_SetFromErrno(PyExc_SystemError);
3756 @@ -250,7 +250,11 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
3759 static PyObject* Database_networks(DatabaseObject* self) {
3760 - return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
3761 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
3764 +static PyObject* Database_networks_flattened(DatabaseObject *self) {
3765 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
3768 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
3769 @@ -264,7 +268,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
3772 struct loc_database_enumerator* enumerator;
3773 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
3774 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
3776 PyErr_SetFromErrno(PyExc_SystemError);
3778 @@ -317,7 +321,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
3781 static PyObject* Database_countries(DatabaseObject* self) {
3782 - return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
3783 + return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
3786 static struct PyMethodDef Database_methods[] = {
3787 @@ -403,6 +407,13 @@ static struct PyGetSetDef Database_getsetters[] = {
3792 + "networks_flattened",
3793 + (getter)Database_networks_flattened,
3800 (getter)Database_get_vendor,
3804 From f5e50a47e37e9b29d0d2ee9e5a41e5a5fe5aea7f Mon Sep 17 00:00:00 2001
3805 From: Michael Tremer <michael.tremer@ipfire.org>
3806 Date: Thu, 12 Nov 2020 19:21:58 +0000
3807 Subject: [PATCH 41/70] network: Reduce debugging output
3809 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3811 src/network.c | 28 +++-------------------------
3812 1 file changed, 3 insertions(+), 25 deletions(-)
3814 diff --git a/src/network.c b/src/network.c
3815 index 44571b3..f7071a6 100644
3818 @@ -421,37 +421,17 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
3821 LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
3822 -#ifdef ENABLE_DEBUG
3823 - char* n1 = loc_network_str(self);
3824 - char* n2 = loc_network_str(other);
3826 - DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
3832 // Family must be the same
3833 - if (self->family != other->family) {
3834 - DEBUG(self->ctx, " Family mismatch\n");
3836 + if (self->family != other->family)
3840 // The start address must be the same
3841 - if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
3842 - DEBUG(self->ctx, " Address mismatch\n");
3844 + if (in6_addr_cmp(&self->first_address, &other->first_address) != 0)
3848 // The prefix length must be the same
3849 - if (self->prefix != other->prefix) {
3850 - DEBUG(self->ctx, " Prefix mismatch\n");
3851 + if (self->prefix != other->prefix)
3855 - DEBUG(self->ctx, " Yes!\n");
3859 @@ -1138,8 +1118,6 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
3860 if (i1 >= list->size || i2 >= list->size)
3863 - DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
3865 struct loc_network* network1 = list->list[i1];
3866 struct loc_network* network2 = list->list[i2];
3871 From 037c65d3a07ec6d37ff063f0645adda6b483b407 Mon Sep 17 00:00:00 2001
3872 From: Michael Tremer <michael.tremer@ipfire.org>
3873 Date: Thu, 12 Nov 2020 19:36:38 +0000
3874 Subject: [PATCH 42/70] python: Export networks exclude function
3876 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3878 src/python/network.c | 39 +++++++++++++++++++++++++++++++++++++++
3879 1 file changed, 39 insertions(+)
3881 diff --git a/src/python/network.c b/src/python/network.c
3882 index 5496d1e..11f672b 100644
3883 --- a/src/python/network.c
3884 +++ b/src/python/network.c
3886 #include "locationmodule.h"
3887 #include "network.h"
3889 +static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
3890 + PyObject* list = PyList_New(0);
3894 + while (!loc_network_list_empty(networks)) {
3895 + struct loc_network* network = loc_network_list_pop(networks);
3897 + PyObject* n = new_network(&NetworkType, network);
3898 + PyList_Append(list, n);
3900 + loc_network_unref(network);
3907 PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
3908 NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
3910 @@ -154,6 +172,21 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
3914 +static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
3915 + NetworkObject* other = NULL;
3917 + if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3920 + struct loc_network_list* list = loc_network_exclude(self->network, other->network);
3922 + // Convert to Python objects
3923 + PyObject* obj = PyList_FromNetworkList(list);
3924 + loc_network_list_unref(list);
3929 static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
3930 NetworkObject* other = NULL;
3932 @@ -191,6 +224,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
3935 static struct PyMethodDef Network_methods[] = {
3938 + (PyCFunction)Network_exclude,
3944 (PyCFunction)Network_has_flag,
3948 From 9a7732c8679e805d4d2d55ea4750c5d70ca4bd2c Mon Sep 17 00:00:00 2001
3949 From: Michael Tremer <michael.tremer@ipfire.org>
3950 Date: Thu, 12 Nov 2020 19:59:22 +0000
3951 Subject: [PATCH 43/70] network: Add more debugging output to stacks
3953 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3955 src/network.c | 16 +++++++++++++---
3956 1 file changed, 13 insertions(+), 3 deletions(-)
3958 diff --git a/src/network.c b/src/network.c
3959 index f7071a6..d41e873 100644
3962 @@ -1088,8 +1088,12 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
3965 // Check if we have space left
3966 - if (list->size == list->max_size)
3967 + if (list->size == list->max_size) {
3968 + ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
3972 + DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
3974 list->list[list->size++] = loc_network_ref(network);
3976 @@ -1098,10 +1102,16 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
3978 LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
3979 // Return nothing when empty
3980 - if (loc_network_list_empty(list))
3981 + if (loc_network_list_empty(list)) {
3982 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
3986 - return list->list[--list->size];
3987 + struct loc_network* network = list->list[--list->size];
3989 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
3994 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
3998 From 33a051e0435f6e78cc936f26f3b9ee16b7851025 Mon Sep 17 00:00:00 2001
3999 From: Michael Tremer <michael.tremer@ipfire.org>
4000 Date: Thu, 12 Nov 2020 20:00:09 +0000
4001 Subject: [PATCH 44/70] network: Add new subnet function
4003 The old one is too difficult to use in terms of order
4004 of input parameters and return value.
4006 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4008 src/libloc.sym | 1 +
4009 src/loc/network.h | 1 +
4010 src/network.c | 15 +++++++++++++++
4011 3 files changed, 17 insertions(+)
4013 diff --git a/src/libloc.sym b/src/libloc.sym
4014 index c0b6b1f..5392437 100644
4015 --- a/src/libloc.sym
4016 +++ b/src/libloc.sym
4017 @@ -87,6 +87,7 @@ global:
4018 loc_network_get_asn;
4019 loc_network_get_country_code;
4020 loc_network_has_flag;
4021 + loc_network_is_subnet;
4022 loc_network_is_subnet_of;
4023 loc_network_match_asn;
4024 loc_network_match_country_code;
4025 diff --git a/src/loc/network.h b/src/loc/network.h
4026 index e30d91c..2154cdc 100644
4027 --- a/src/loc/network.h
4028 +++ b/src/loc/network.h
4029 @@ -56,6 +56,7 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
4031 int loc_network_eq(struct loc_network* self, struct loc_network* other);
4032 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
4033 +int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
4034 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
4035 struct loc_network_list* loc_network_subnets(struct loc_network* network);
4036 struct loc_network_list* loc_network_exclude(
4037 diff --git a/src/network.c b/src/network.c
4038 index d41e873..5719111 100644
4041 @@ -479,6 +479,21 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
4045 +LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
4046 + // If the start address of the other network is smaller than this network,
4047 + // it cannot be a subnet.
4048 + if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
4051 + // If the end address of the other network is greater than this network,
4052 + // it cannot be a subnet.
4053 + if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
4059 +// XXX DEPRECATED - I find this too difficult to use
4060 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
4061 // If the start address of the other network is smaller than this network,
4062 // it cannot be a subnet.
4066 From add5bb652ba1dad1127f79cb6a0db2d363a6d5e5 Mon Sep 17 00:00:00 2001
4067 From: Michael Tremer <michael.tremer@ipfire.org>
4068 Date: Thu, 12 Nov 2020 20:01:17 +0000
4069 Subject: [PATCH 45/70] network: Add function to exclude multiple networks at
4072 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4074 src/libloc.sym | 1 +
4075 src/loc/network.h | 2 ++
4076 src/network.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++
4077 3 files changed, 88 insertions(+)
4079 diff --git a/src/libloc.sym b/src/libloc.sym
4080 index 5392437..bcd11be 100644
4081 --- a/src/libloc.sym
4082 +++ b/src/libloc.sym
4083 @@ -82,6 +82,7 @@ global:
4084 loc_network_address_family;
4086 loc_network_exclude;
4087 + loc_network_exclude_list;
4088 loc_network_format_first_address;
4089 loc_network_format_last_address;
4090 loc_network_get_asn;
4091 diff --git a/src/loc/network.h b/src/loc/network.h
4092 index 2154cdc..40712b9 100644
4093 --- a/src/loc/network.h
4094 +++ b/src/loc/network.h
4095 @@ -61,6 +61,8 @@ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other
4096 struct loc_network_list* loc_network_subnets(struct loc_network* network);
4097 struct loc_network_list* loc_network_exclude(
4098 struct loc_network* self, struct loc_network* other);
4099 +struct loc_network_list* loc_network_exclude_list(
4100 + struct loc_network* network, struct loc_network_list* list);
4103 struct loc_network_list;
4104 diff --git a/src/network.c b/src/network.c
4105 index 5719111..751e8e5 100644
4108 @@ -677,6 +677,91 @@ ERROR:
4112 +LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
4113 + struct loc_network* network, struct loc_network_list* list) {
4114 + struct loc_network_list* to_check;
4116 + // Create a new list with all networks to look at
4117 + int r = loc_network_list_new(network->ctx, &to_check);
4121 + struct loc_network* subnet = NULL;
4122 + struct loc_network_list* subnets = NULL;
4124 + for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
4125 + subnet = loc_network_list_get(list, i);
4127 + // Find all excluded networks
4128 + struct loc_network_list* excluded = loc_network_exclude(network, subnet);
4130 + // Add them all to the "to check" list
4131 + loc_network_list_merge(to_check, excluded);
4132 + loc_network_list_unref(excluded);
4136 + loc_network_unref(subnet);
4139 + r = loc_network_list_new(network->ctx, &subnets);
4141 + loc_network_list_unref(to_check);
4145 + while (!loc_network_list_empty(to_check)) {
4146 + struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
4148 + // Marks whether this subnet passed all checks
4151 + for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
4152 + subnet = loc_network_list_get(list, i);
4154 + // Drop this subnet if is is already in list
4155 + if (loc_network_eq(subnet_to_check, subnet)) {
4157 + loc_network_unref(subnet);
4161 + // Drop this subnet if is a subnet of another subnet
4162 + if (loc_network_is_subnet_of(subnet, subnet_to_check)) {
4164 + loc_network_unref(subnet);
4168 + // Break it down if it overlaps
4169 + if (loc_network_overlaps(subnet_to_check, subnet)) {
4172 + struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet);
4174 + loc_network_list_merge(to_check, excluded);
4175 + loc_network_list_unref(excluded);
4178 + loc_network_unref(subnet);
4182 + loc_network_unref(subnet);
4186 + r = loc_network_list_push(subnets, subnet_to_check);
4189 + loc_network_unref(subnet_to_check);
4192 + loc_network_list_unref(to_check);
4197 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
4199 loc_country_code_copy(dbobj->country_code, network->country_code);
4203 From d87fd7a3d277b4b03222c7d1680e51b3e45e525b Mon Sep 17 00:00:00 2001
4204 From: Michael Tremer <michael.tremer@ipfire.org>
4205 Date: Thu, 12 Nov 2020 20:02:03 +0000
4206 Subject: [PATCH 46/70] database: Add option to return networks flattened
4208 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4210 src/database.c | 174 ++++++++++++++++++++++++++++++++++++++++---------
4211 1 file changed, 143 insertions(+), 31 deletions(-)
4213 diff --git a/src/database.c b/src/database.c
4214 index 9baab33..7a3d1a7 100644
4215 --- a/src/database.c
4216 +++ b/src/database.c
4217 @@ -118,6 +118,9 @@ struct loc_database_enumerator {
4218 struct loc_node_stack network_stack[MAX_STACK_DEPTH];
4219 int network_stack_depth;
4220 unsigned int* networks_visited;
4222 + // For subnet search
4223 + struct loc_network_list* stack;
4226 static int loc_database_read_magic(struct loc_database* db) {
4227 @@ -935,6 +938,26 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
4231 +static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
4232 + DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
4234 + // Release all references
4235 + loc_database_unref(enumerator->db);
4236 + loc_unref(enumerator->ctx);
4238 + if (enumerator->string)
4239 + free(enumerator->string);
4241 + // Free network search
4242 + free(enumerator->networks_visited);
4244 + // Free subnet stack
4245 + if (enumerator->stack)
4246 + loc_network_list_unref(enumerator->stack);
4251 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
4252 struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
4253 struct loc_database_enumerator* e = calloc(1, sizeof(*e));
4254 @@ -951,10 +974,16 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
4255 e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
4257 // Initialise graph search
4258 - //e->network_stack[++e->network_stack_depth] = 0;
4259 e->network_stack_depth = 1;
4260 e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
4263 + int r = loc_network_list_new(e->ctx, &e->stack);
4265 + loc_database_enumerator_free(e);
4269 DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
4272 @@ -967,22 +996,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
4276 -static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
4277 - DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
4279 - // Release all references
4280 - loc_database_unref(enumerator->db);
4281 - loc_unref(enumerator->ctx);
4283 - if (enumerator->string)
4284 - free(enumerator->string);
4286 - // Free network search
4287 - free(enumerator->networks_visited);
4292 LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
4295 @@ -1116,17 +1129,13 @@ static int loc_database_enumerator_stack_push_node(
4299 -LOC_EXPORT int loc_database_enumerator_next_network(
4300 - struct loc_database_enumerator* enumerator, struct loc_network** network) {
4304 - // Do not do anything if not in network mode
4305 - if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
4306 +static int __loc_database_enumerator_next_network(
4307 + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
4308 + // Return top element from the stack
4309 + *network = loc_network_list_pop(enumerator->stack);
4315 DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
4316 enumerator->network_stack_depth);
4318 @@ -1155,7 +1164,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
4319 enumerator->db->network_nodes_v1 + node->offset;
4321 // Add edges to stack
4322 - r = loc_database_enumerator_stack_push_node(enumerator,
4323 + int r = loc_database_enumerator_stack_push_node(enumerator,
4324 be32toh(n->one), 1, node->depth + 1);
4327 @@ -1181,6 +1190,10 @@ LOC_EXPORT int loc_database_enumerator_next_network(
4331 + // Return all networks when the filter is disabled
4335 // Check if we are interested in this network
4337 // Skip if the family does not match
4338 @@ -1223,12 +1236,111 @@ LOC_EXPORT int loc_database_enumerator_next_network(
4341 // Reached the end of the search
4345 - // Mark all nodes as non-visited
4346 - for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
4347 - enumerator->networks_visited[i] = 0;
4348 +static int __loc_database_enumerator_next_network_flattened(
4349 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
4350 + // Fetch the next network
4351 + int r = __loc_database_enumerator_next_network(enumerator, network, 1);
4356 + // End if we could not read another network
4360 + struct loc_network* subnet = NULL;
4361 + struct loc_network_list* subnets;
4363 + // Create a list with all subnets
4364 + r = loc_network_list_new(enumerator->ctx, &subnets);
4368 + // Search all subnets from the database
4370 + // Fetch the next network in line
4371 + r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
4375 + // End if we did not receive another subnet
4379 + // Collect all subnets in a list
4380 + if (loc_network_is_subnet(*network, subnet)) {
4381 + r = loc_network_list_push(subnets, subnet);
4385 + loc_network_unref(subnet);
4389 + // If this is not a subnet, we push it back onto the stack and break
4390 + r = loc_network_list_push(enumerator->stack, subnet);
4394 + loc_network_unref(subnet);
4398 + DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
4400 + // We can abort here if the network has no subnets
4401 + if (loc_network_list_empty(subnets)) {
4402 + loc_network_list_unref(subnets);
4407 + // If the network has any subnets, we will break it into smaller parts
4408 + // without the subnets.
4409 + struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
4410 + if (!excluded || loc_network_list_empty(excluded)) {
4415 + // Sort the result
4416 + loc_network_list_sort(excluded);
4418 + // Reverse the list
4419 + loc_network_list_reverse(excluded);
4421 + // Replace network with the first one
4422 + loc_network_unref(*network);
4424 + *network = loc_network_list_pop(excluded);
4426 + // Push the rest onto the stack
4427 + loc_network_list_merge(enumerator->stack, excluded);
4429 + loc_network_list_unref(excluded);
4433 + loc_network_unref(subnet);
4435 + loc_network_list_unref(subnets);
4440 +LOC_EXPORT int loc_database_enumerator_next_network(
4441 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
4442 + // Do not do anything if not in network mode
4443 + if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
4446 + // Flatten output?
4447 + if (enumerator->flatten)
4448 + return __loc_database_enumerator_next_network_flattened(enumerator, network);
4450 + return __loc_database_enumerator_next_network(enumerator, network, 1);
4453 LOC_EXPORT int loc_database_enumerator_next_country(
4457 From d3ae93c27dcd7f6984fdc29cc141621e277f2e2a Mon Sep 17 00:00:00 2001
4458 From: Michael Tremer <michael.tremer@ipfire.org>
4459 Date: Thu, 12 Nov 2020 20:09:20 +0000
4460 Subject: [PATCH 47/70] test: Update API
4462 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4464 src/test-as.c | 2 +-
4465 src/test-database.c | 2 +-
4466 2 files changed, 2 insertions(+), 2 deletions(-)
4468 diff --git a/src/test-as.c b/src/test-as.c
4469 index 839a04c..2d61675 100644
4472 @@ -95,7 +95,7 @@ int main(int argc, char** argv) {
4475 struct loc_database_enumerator* enumerator;
4476 - err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
4477 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
4479 fprintf(stderr, "Could not create a database enumerator\n");
4481 diff --git a/src/test-database.c b/src/test-database.c
4482 index 4aef94e..da4b11c 100644
4483 --- a/src/test-database.c
4484 +++ b/src/test-database.c
4485 @@ -198,7 +198,7 @@ int main(int argc, char** argv) {
4488 struct loc_database_enumerator* enumerator;
4489 - err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
4490 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
4492 fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
4497 From 594ca328c6e124d0f1eb543e9c8d9bbfe8a7b628 Mon Sep 17 00:00:00 2001
4498 From: Michael Tremer <michael.tremer@ipfire.org>
4499 Date: Thu, 12 Nov 2020 20:09:37 +0000
4500 Subject: [PATCH 48/70] networks: Copy all attributes when splitting networks
4502 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4504 src/network.c | 14 ++++++++++++++
4505 1 file changed, 14 insertions(+)
4507 diff --git a/src/network.c b/src/network.c
4508 index 751e8e5..d67f116 100644
4511 @@ -550,6 +550,20 @@ LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* netw
4515 + // Copy country code
4516 + const char* country_code = loc_network_get_country_code(network);
4517 + if (country_code) {
4518 + loc_network_set_country_code(subnet1, country_code);
4519 + loc_network_set_country_code(subnet2, country_code);
4523 + uint32_t asn = loc_network_get_asn(network);
4525 + loc_network_set_asn(subnet1, asn);
4526 + loc_network_set_asn(subnet2, asn);
4529 loc_network_unref(subnet1);
4530 loc_network_unref(subnet2);
4535 From 69248038292e9ea1a4ee8912cdfc8700456753ad Mon Sep 17 00:00:00 2001
4536 From: Michael Tremer <michael.tremer@ipfire.org>
4537 Date: Fri, 13 Nov 2020 11:23:33 +0000
4538 Subject: [PATCH 49/70] database: Move network filtering into a separate
4541 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4543 src/database.c | 56 +++++++++++++++++++++++---------------------------
4544 1 file changed, 26 insertions(+), 30 deletions(-)
4546 diff --git a/src/database.c b/src/database.c
4547 index 7a3d1a7..72bc8eb 100644
4548 --- a/src/database.c
4549 +++ b/src/database.c
4550 @@ -1129,6 +1129,31 @@ static int loc_database_enumerator_stack_push_node(
4554 +static int loc_database_enumerator_filter_network(
4555 + struct loc_database_enumerator* enumerator, struct loc_network* network) {
4556 + // Skip if the family does not match
4557 + if (enumerator->family && loc_network_address_family(network) != enumerator->family)
4560 + // Skip if the country code does not match
4561 + if (*enumerator->country_code &&
4562 + !loc_network_match_country_code(network, enumerator->country_code))
4565 + // Skip if the ASN does not match
4566 + if (enumerator->asn &&
4567 + !loc_network_match_asn(network, enumerator->asn))
4570 + // Skip if flags do not match
4571 + if (enumerator->flags &&
4572 + !loc_network_match_flag(network, enumerator->flags))
4579 static int __loc_database_enumerator_next_network(
4580 struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
4581 // Return top element from the stack
4582 @@ -1195,36 +1220,7 @@ static int __loc_database_enumerator_next_network(
4585 // Check if we are interested in this network
4587 - // Skip if the family does not match
4588 - if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
4589 - loc_network_unref(*network);
4595 - // Skip if the country code does not match
4596 - if (*enumerator->country_code &&
4597 - !loc_network_match_country_code(*network, enumerator->country_code)) {
4598 - loc_network_unref(*network);
4604 - // Skip if the ASN does not match
4605 - if (enumerator->asn &&
4606 - !loc_network_match_asn(*network, enumerator->asn)) {
4607 - loc_network_unref(*network);
4613 - // Skip if flags do not match
4614 - if (enumerator->flags &&
4615 - !loc_network_match_flag(*network, enumerator->flags)) {
4616 + if (loc_database_enumerator_filter_network(enumerator, *network)) {
4617 loc_network_unref(*network);
4623 From 2113e71bf7b997c82670c5c22cf91aa6442fe6f3 Mon Sep 17 00:00:00 2001
4624 From: Michael Tremer <michael.tremer@ipfire.org>
4625 Date: Fri, 13 Nov 2020 11:29:02 +0000
4626 Subject: [PATCH 50/70] database: Filter results coming from stack
4628 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4630 src/database.c | 18 ++++++++++++++++--
4631 1 file changed, 16 insertions(+), 2 deletions(-)
4633 diff --git a/src/database.c b/src/database.c
4634 index 72bc8eb..0f3cdc2 100644
4635 --- a/src/database.c
4636 +++ b/src/database.c
4637 @@ -1157,9 +1157,23 @@ static int loc_database_enumerator_filter_network(
4638 static int __loc_database_enumerator_next_network(
4639 struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
4640 // Return top element from the stack
4641 - *network = loc_network_list_pop(enumerator->stack);
4644 + *network = loc_network_list_pop(enumerator->stack);
4650 + // Throw away any networks by filter
4651 + if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
4652 + loc_network_unref(*network);
4661 DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
4662 enumerator->network_stack_depth);
4666 From d33753688138c9938743dafbbdddf220dd2afd14 Mon Sep 17 00:00:00 2001
4667 From: Michael Tremer <michael.tremer@ipfire.org>
4668 Date: Fri, 13 Nov 2020 11:29:15 +0000
4669 Subject: [PATCH 51/70] network: Sort result of excluded lists
4671 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4673 src/database.c | 3 ---
4674 src/network.c | 3 +++
4675 2 files changed, 3 insertions(+), 3 deletions(-)
4677 diff --git a/src/database.c b/src/database.c
4678 index 0f3cdc2..6849d97 100644
4679 --- a/src/database.c
4680 +++ b/src/database.c
4681 @@ -1315,9 +1315,6 @@ static int __loc_database_enumerator_next_network_flattened(
4685 - // Sort the result
4686 - loc_network_list_sort(excluded);
4689 loc_network_list_reverse(excluded);
4691 diff --git a/src/network.c b/src/network.c
4692 index d67f116..9d02bf8 100644
4695 @@ -773,6 +773,9 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
4697 loc_network_list_unref(to_check);
4699 + // Sort the result
4700 + loc_network_list_sort(subnets);
4708 From 8d777f12f7ffa4df1b28d197563888296803b727 Mon Sep 17 00:00:00 2001
4709 From: Michael Tremer <michael.tremer@ipfire.org>
4710 Date: Fri, 13 Nov 2020 11:38:15 +0000
4711 Subject: [PATCH 52/70] network: Add function to pop first element from stack
4713 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4715 src/database.c | 6 ++----
4716 src/libloc.sym | 1 +
4717 src/loc/network.h | 1 +
4718 src/network.c | 19 +++++++++++++++++++
4719 4 files changed, 23 insertions(+), 4 deletions(-)
4721 diff --git a/src/database.c b/src/database.c
4722 index 6849d97..b9d870f 100644
4723 --- a/src/database.c
4724 +++ b/src/database.c
4725 @@ -1315,15 +1315,13 @@ static int __loc_database_enumerator_next_network_flattened(
4729 - // Reverse the list
4730 - loc_network_list_reverse(excluded);
4732 // Replace network with the first one
4733 loc_network_unref(*network);
4735 - *network = loc_network_list_pop(excluded);
4736 + *network = loc_network_list_pop_first(excluded);
4738 // Push the rest onto the stack
4739 + loc_network_list_reverse(excluded);
4740 loc_network_list_merge(enumerator->stack, excluded);
4742 loc_network_list_unref(excluded);
4743 diff --git a/src/libloc.sym b/src/libloc.sym
4744 index bcd11be..6139db6 100644
4745 --- a/src/libloc.sym
4746 +++ b/src/libloc.sym
4747 @@ -113,6 +113,7 @@ global:
4748 loc_network_list_merge;
4749 loc_network_list_new;
4750 loc_network_list_pop;
4751 + loc_network_list_pop_first;
4752 loc_network_list_push;
4753 loc_network_list_ref;
4754 loc_network_list_reverse;
4755 diff --git a/src/loc/network.h b/src/loc/network.h
4756 index 40712b9..203e61c 100644
4757 --- a/src/loc/network.h
4758 +++ b/src/loc/network.h
4759 @@ -76,6 +76,7 @@ void loc_network_list_dump(struct loc_network_list* list);
4760 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
4761 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
4762 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
4763 +struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
4764 int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
4765 void loc_network_list_sort(struct loc_network_list* list);
4766 void loc_network_list_reverse(struct loc_network_list* list);
4767 diff --git a/src/network.c b/src/network.c
4768 index 9d02bf8..e7dc97e 100644
4771 @@ -1231,6 +1231,25 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
4775 +LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
4776 + // Return nothing when empty
4777 + if (loc_network_list_empty(list)) {
4778 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
4782 + struct loc_network* network = list->list[0];
4784 + // Move all elements to the top of the stack
4785 + for (unsigned int i = 0; i < --list->size; i++) {
4786 + list->list[i] = list->list[i+1];
4789 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
4794 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
4795 for (unsigned int i = 0; i < list->size; i++) {
4796 if (loc_network_eq(list->list[i], network))
4800 From 7933f5bfb4dd7603cb646e192840762bf6394292 Mon Sep 17 00:00:00 2001
4801 From: Michael Tremer <michael.tremer@ipfire.org>
4802 Date: Fri, 13 Nov 2020 11:43:53 +0000
4803 Subject: [PATCH 53/70] network: Unexport all tree functions
4805 These should not be exported
4807 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4809 src/network.c | 28 ++++++++++++++--------------
4810 src/test-network.c | 9 +++++++++
4811 2 files changed, 23 insertions(+), 14 deletions(-)
4813 diff --git a/src/network.c b/src/network.c
4814 index e7dc97e..d015579 100644
4817 @@ -847,7 +847,7 @@ struct loc_network_tree_node {
4818 struct loc_network* network;
4821 -LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
4822 +int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
4823 struct loc_network_tree* t = calloc(1, sizeof(*t));
4826 @@ -867,7 +867,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
4830 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
4831 +struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
4832 return loc_network_tree_node_ref(tree->root);
4835 @@ -939,7 +939,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
4839 -LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
4840 +int loc_network_tree_walk(struct loc_network_tree* tree,
4841 int(*filter_callback)(struct loc_network* network, void* data),
4842 int(*callback)(struct loc_network* network, void* data), void* data) {
4843 return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
4844 @@ -954,7 +954,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
4848 -LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
4849 +struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
4850 if (--tree->refcount > 0)
4853 @@ -975,13 +975,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
4857 -LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
4858 +int loc_network_tree_dump(struct loc_network_tree* tree) {
4859 DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
4861 return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
4864 -LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
4865 +int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
4866 DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
4868 struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
4869 @@ -1012,7 +1012,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
4873 -LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
4874 +size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
4877 int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
4878 @@ -1034,11 +1034,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
4882 -LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
4883 +size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
4884 return __loc_network_tree_count_nodes(tree->root);
4887 -LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
4888 +int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
4889 struct loc_network_tree_node* n = calloc(1, sizeof(*n));
4892 @@ -1053,7 +1053,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
4896 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
4897 +struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
4901 @@ -1076,7 +1076,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
4905 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
4906 +struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
4910 @@ -1087,7 +1087,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
4914 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
4915 +struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
4919 @@ -1099,11 +1099,11 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
4920 return loc_network_tree_node_ref(node);
4923 -LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
4924 +int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
4925 return (!!node->network);
4928 -LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
4929 +struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
4930 return loc_network_ref(node->network);
4933 diff --git a/src/test-network.c b/src/test-network.c
4934 index af1b2e6..7c90224 100644
4935 --- a/src/test-network.c
4936 +++ b/src/test-network.c
4937 @@ -37,12 +37,14 @@ int main(int argc, char** argv) {
4938 // Enable debug logging
4939 loc_set_log_priority(ctx, LOG_DEBUG);
4942 struct loc_network_tree* tree;
4943 err = loc_network_tree_new(ctx, &tree);
4945 fprintf(stderr, "Could not create the network tree\n");
4951 struct loc_network* network1;
4952 @@ -58,12 +60,14 @@ int main(int argc, char** argv) {
4957 // Adding network to the tree
4958 err = loc_network_tree_add_network(tree, network1);
4960 fprintf(stderr, "Could not add network to the tree\n");
4965 // Check if the first and last addresses are correct
4966 char* string = loc_network_format_first_address(network1);
4967 @@ -101,6 +105,7 @@ int main(int argc, char** argv) {
4972 // Adding network to the tree
4973 err = loc_network_tree_add_network(tree, network2);
4975 @@ -117,6 +122,7 @@ int main(int argc, char** argv) {
4977 size_t nodes = loc_network_tree_count_nodes(tree);
4978 printf("The tree has %zu nodes\n", nodes);
4981 // Check equals function
4982 err = loc_network_eq(network1, network1);
4983 @@ -260,7 +266,10 @@ int main(int argc, char** argv) {
4984 loc_network_unref(network2);
4985 loc_network_unref(network3);
4986 loc_network_unref(network4);
4989 loc_network_tree_unref(tree);
4992 // And open it again from disk
4993 struct loc_database* db;
4997 From c242f7325bd6fc4ba26047ac24196d1c161c6e01 Mon Sep 17 00:00:00 2001
4998 From: Michael Tremer <michael.tremer@ipfire.org>
4999 Date: Fri, 13 Nov 2020 12:09:03 +0000
5000 Subject: [PATCH 54/70] python: Move tree flattening into C
5002 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
5004 src/python/database.c | 8 ++-
5005 src/python/export.py | 138 +++++-------------------------------------
5006 2 files changed, 21 insertions(+), 125 deletions(-)
5008 diff --git a/src/python/database.c b/src/python/database.c
5009 index 7f8c2c2..d169547 100644
5010 --- a/src/python/database.c
5011 +++ b/src/python/database.c
5012 @@ -258,17 +258,19 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
5015 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
5016 - char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
5017 + char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
5018 const char* country_code = NULL;
5019 unsigned int asn = 0;
5024 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
5025 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
5028 struct loc_database_enumerator* enumerator;
5029 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
5030 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
5031 + (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
5033 PyErr_SetFromErrno(PyExc_SystemError);
5035 diff --git a/src/python/export.py b/src/python/export.py
5036 index dd44332..be4a68e 100644
5037 --- a/src/python/export.py
5038 +++ b/src/python/export.py
5039 @@ -87,58 +87,12 @@ class OutputWriter(object):
5040 def _write_network(self, network):
5041 self.f.write("%s\n" % network)
5043 - def write(self, network, subnets):
5044 + def write(self, network):
5045 if self.flatten and self._flatten(network):
5046 log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
5049 - # Convert network into a Python object
5050 - _network = ipaddress.ip_network("%s" % network)
5052 - # Write the network when it has no subnets
5054 - log.debug("Writing %s to %s" % (_network, self.f))
5055 - return self._write_network(_network)
5057 - # Convert subnets into Python objects
5058 - _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
5060 - # Split the network into smaller bits so that
5061 - # we can accomodate for any gaps in it later
5063 - for _subnet in _subnets:
5065 - _network.address_exclude(_subnet)
5068 - # Clear the list of all subnets
5071 - # Check if all subnets to not overlap with anything else
5073 - subnet_to_check = to_check.pop()
5075 - for _subnet in _subnets:
5076 - # Drop this subnet if it equals one of the subnets
5077 - # or if it is subnet of one of them
5078 - if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
5081 - # Break it down if it overlaps
5082 - if subnet_to_check.overlaps(_subnet):
5084 - subnet_to_check.address_exclude(_subnet)
5088 - # Add the subnet again as it passed the check
5090 - subnets.append(subnet_to_check)
5092 - # Write all networks as compact as possible
5093 - for network in ipaddress.collapse_addresses(subnets):
5094 - log.debug("Writing %s to %s" % (network, self.f))
5095 - self._write_network(network)
5096 + return self._write_network(network)
5100 @@ -188,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter):
5103 def _write_network(self, network):
5104 - for address in (network.network_address, network.broadcast_address):
5105 + for address in (network.first_address, network.last_address):
5106 # Convert this into a string of bits
5107 bytes = socket.inet_pton(
5108 socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
5109 @@ -231,42 +185,21 @@ class Exporter(object):
5110 writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
5112 # Get all networks that match the family
5113 - networks = self.db.search_networks(family=family)
5115 - # Create a stack with all networks in order where we can put items back
5116 - # again and retrieve them in the next iteration.
5117 - networks = BufferedStack(networks)
5118 + networks = self.db.search_networks(family=family, flatten=True)
5120 # Walk through all networks
5121 for network in networks:
5122 - # Collect all networks which are a subnet of network
5124 - for subnet in networks:
5125 - # If the next subnet was not a subnet, we have to push
5126 - # it back on the stack and break this loop
5127 - if not subnet.is_subnet_of(network):
5128 - networks.push(subnet)
5131 - subnets.append(subnet)
5133 # Write matching countries
5134 - if network.country_code and network.country_code in writers:
5135 - # Mismatching subnets
5137 - subnet for subnet in subnets if not network.country_code == subnet.country_code
5140 - writers[network.country_code].write(network, gaps)
5142 + writers[network.country_code].write(network)
5146 # Write matching ASNs
5147 - if network.asn and network.asn in writers:
5148 - # Mismatching subnets
5150 - subnet for subnet in subnets if not network.asn == subnet.asn
5153 - writers[network.asn].write(network, gaps)
5155 + writers[network.asn].write(network)
5161 @@ -274,19 +207,10 @@ class Exporter(object):
5162 # Fetch the "fake" country code
5163 country = flags[flag]
5165 - if not country in writers:
5169 - subnet for subnet in subnets
5170 - if not subnet.has_flag(flag)
5173 - writers[country].write(network, gaps)
5175 - # Push all subnets back onto the stack
5176 - for subnet in reversed(subnets):
5177 - networks.push(subnet)
5179 + writers[country].write(network)
5183 # Write everything to the filesystem
5184 for writer in writers.values():
5185 @@ -298,33 +222,3 @@ class Exporter(object):
5188 return os.path.join(directory, filename)
5191 -class BufferedStack(object):
5193 - This class takes an iterator and when being iterated
5194 - over it returns objects from that iterator for as long
5197 - It additionally has a function to put an item back on
5198 - the back so that it will be returned again at the next
5201 - def __init__(self, iterator):
5202 - self.iterator = iterator
5205 - def __iter__(self):
5208 - def __next__(self):
5210 - return self.stack.pop(0)
5212 - return next(self.iterator)
5214 - def push(self, elem):
5216 - Takes an element and puts it on the stack
5218 - self.stack.insert(0, elem)
5222 From e0b9ff5f38beb0d560b16db881647e5a75127df1 Mon Sep 17 00:00:00 2001
5223 From: Michael Tremer <michael.tremer@ipfire.org>
5224 Date: Sun, 15 Nov 2020 15:02:28 +0000
5225 Subject: [PATCH 55/70] Move network lists into an own file
5227 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
5230 src/libloc.sym | 1 +
5231 src/loc/network-list.h | 37 +++++++
5232 src/loc/network.h | 20 +---
5233 src/network-list.c | 224 +++++++++++++++++++++++++++++++++++++++++
5234 src/network.c | 207 +------------------------------------
5235 src/python/network.c | 1 +
5236 7 files changed, 269 insertions(+), 223 deletions(-)
5237 create mode 100644 src/loc/network-list.h
5238 create mode 100644 src/network-list.c
5240 diff --git a/Makefile.am b/Makefile.am
5241 index a0431a6..f0d8c4c 100644
5244 @@ -96,6 +96,7 @@ pkginclude_HEADERS = \
5245 src/loc/database.h \
5248 + src/loc/network-list.h \
5250 src/loc/stringpool.h \
5252 @@ -110,6 +111,7 @@ src_libloc_la_SOURCES = \
5256 + src/network-list.c \
5260 diff --git a/src/libloc.sym b/src/libloc.sym
5261 index 6139db6..453a1be 100644
5262 --- a/src/libloc.sym
5263 +++ b/src/libloc.sym
5264 @@ -87,6 +87,7 @@ global:
5265 loc_network_format_last_address;
5266 loc_network_get_asn;
5267 loc_network_get_country_code;
5269 loc_network_has_flag;
5270 loc_network_is_subnet;
5271 loc_network_is_subnet_of;
5272 diff --git a/src/loc/network-list.h b/src/loc/network-list.h
5273 new file mode 100644
5274 index 0000000..af3b28d
5276 +++ b/src/loc/network-list.h
5279 + libloc - A library to determine the location of someone on the Internet
5281 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
5283 + This library is free software; you can redistribute it and/or
5284 + modify it under the terms of the GNU Lesser General Public
5285 + License as published by the Free Software Foundation; either
5286 + version 2.1 of the License, or (at your option) any later version.
5288 + This library is distributed in the hope that it will be useful,
5289 + but WITHOUT ANY WARRANTY; without even the implied warranty of
5290 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5291 + Lesser General Public License for more details.
5294 +#ifndef LIBLOC_NETWORK_LIST_H
5295 +#define LIBLOC_NETWORK_LIST_H
5297 +struct loc_network_list;
5298 +int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
5299 +struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
5300 +struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
5301 +size_t loc_network_list_size(struct loc_network_list* list);
5302 +int loc_network_list_empty(struct loc_network_list* list);
5303 +void loc_network_list_clear(struct loc_network_list* list);
5304 +void loc_network_list_dump(struct loc_network_list* list);
5305 +struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
5306 +int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
5307 +struct loc_network* loc_network_list_pop(struct loc_network_list* list);
5308 +struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
5309 +int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
5310 +void loc_network_list_sort(struct loc_network_list* list);
5311 +void loc_network_list_reverse(struct loc_network_list* list);
5312 +int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
5315 diff --git a/src/loc/network.h b/src/loc/network.h
5316 index 203e61c..d86b685 100644
5317 --- a/src/loc/network.h
5318 +++ b/src/loc/network.h
5321 #include <loc/libloc.h>
5322 #include <loc/format.h>
5323 +#include <loc/network-list.h>
5325 enum loc_network_flags {
5326 LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
5327 @@ -55,6 +56,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
5328 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
5330 int loc_network_eq(struct loc_network* self, struct loc_network* other);
5331 +int loc_network_gt(struct loc_network* self, struct loc_network* other);
5332 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
5333 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
5334 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
5335 @@ -64,24 +66,6 @@ struct loc_network_list* loc_network_exclude(
5336 struct loc_network_list* loc_network_exclude_list(
5337 struct loc_network* network, struct loc_network_list* list);
5340 -struct loc_network_list;
5341 -int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
5342 -struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
5343 -struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
5344 -size_t loc_network_list_size(struct loc_network_list* list);
5345 -int loc_network_list_empty(struct loc_network_list* list);
5346 -void loc_network_list_clear(struct loc_network_list* list);
5347 -void loc_network_list_dump(struct loc_network_list* list);
5348 -struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
5349 -int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
5350 -struct loc_network* loc_network_list_pop(struct loc_network_list* list);
5351 -struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
5352 -int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
5353 -void loc_network_list_sort(struct loc_network_list* list);
5354 -void loc_network_list_reverse(struct loc_network_list* list);
5355 -int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
5357 #ifdef LIBLOC_PRIVATE
5359 int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
5360 diff --git a/src/network-list.c b/src/network-list.c
5361 new file mode 100644
5362 index 0000000..1f6e80e
5364 +++ b/src/network-list.c
5367 + libloc - A library to determine the location of someone on the Internet
5369 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
5371 + This library is free software; you can redistribute it and/or
5372 + modify it under the terms of the GNU Lesser General Public
5373 + License as published by the Free Software Foundation; either
5374 + version 2.1 of the License, or (at your option) any later version.
5376 + This library is distributed in the hope that it will be useful,
5377 + but WITHOUT ANY WARRANTY; without even the implied warranty of
5378 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5379 + Lesser General Public License for more details.
5383 +#include <stdlib.h>
5385 +#include <loc/libloc.h>
5386 +#include <loc/network.h>
5387 +#include <loc/private.h>
5389 +struct loc_network_list {
5390 + struct loc_ctx* ctx;
5393 + struct loc_network* list[1024];
5398 +LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
5399 + struct loc_network_list** list) {
5400 + struct loc_network_list* l = calloc(1, sizeof(*l));
5404 + l->ctx = loc_ref(ctx);
5407 + // Do not allow this list to grow larger than this
5408 + l->max_size = 1024;
5410 + DEBUG(l->ctx, "Network list allocated at %p\n", l);
5415 +LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
5421 +static void loc_network_list_free(struct loc_network_list* list) {
5422 + DEBUG(list->ctx, "Releasing network list at %p\n", list);
5424 + for (unsigned int i = 0; i < list->size; i++)
5425 + loc_network_unref(list->list[i]);
5427 + loc_unref(list->ctx);
5431 +LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
5435 + if (--list->refcount > 0)
5438 + loc_network_list_free(list);
5442 +LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
5443 + return list->size;
5446 +LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
5447 + return list->size == 0;
5450 +LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
5451 + for (unsigned int i = 0; i < list->size; i++)
5452 + loc_network_unref(list->list[i]);
5457 +LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
5458 + struct loc_network* network;
5461 + for (unsigned int i = 0; i < list->size; i++) {
5462 + network = list->list[i];
5464 + s = loc_network_str(network);
5466 + INFO(list->ctx, "%s\n", s);
5471 +LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
5473 + if (index >= list->size)
5476 + return loc_network_ref(list->list[index]);
5479 +LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
5480 + // Do not add networks that are already on the list
5481 + if (loc_network_list_contains(list, network))
5484 + // Check if we have space left
5485 + if (list->size == list->max_size) {
5486 + ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
5490 + DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
5492 + list->list[list->size++] = loc_network_ref(network);
5497 +LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
5498 + // Return nothing when empty
5499 + if (loc_network_list_empty(list)) {
5500 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5504 + struct loc_network* network = list->list[--list->size];
5506 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5511 +LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
5512 + // Return nothing when empty
5513 + if (loc_network_list_empty(list)) {
5514 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5518 + struct loc_network* network = list->list[0];
5520 + // Move all elements to the top of the stack
5521 + for (unsigned int i = 0; i < --list->size; i++) {
5522 + list->list[i] = list->list[i+1];
5525 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5530 +LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
5531 + for (unsigned int i = 0; i < list->size; i++) {
5532 + if (loc_network_eq(list->list[i], network))
5539 +static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
5540 + // Do nothing for invalid indices
5541 + if (i1 >= list->size || i2 >= list->size)
5544 + struct loc_network* network1 = list->list[i1];
5545 + struct loc_network* network2 = list->list[i2];
5547 + list->list[i1] = network2;
5548 + list->list[i2] = network1;
5551 +LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
5552 + unsigned int i = 0;
5553 + unsigned int j = list->size - 1;
5556 + loc_network_list_swap(list, i++, j--);
5560 +LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
5561 + unsigned int n = list->size;
5567 + for (unsigned int i = 1; i < n; i++) {
5568 + if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
5569 + loc_network_list_swap(list, i-1, i);
5575 + } while (swapped);
5578 +LOC_EXPORT int loc_network_list_merge(
5579 + struct loc_network_list* self, struct loc_network_list* other) {
5582 + for (unsigned int i = 0; i < other->size; i++) {
5583 + r = loc_network_list_push(self, other->list[i]);
5590 diff --git a/src/network.c b/src/network.c
5591 index d015579..28ca2df 100644
5595 #include <loc/compat.h>
5596 #include <loc/country.h>
5597 #include <loc/network.h>
5598 +#include <loc/network-list.h>
5599 #include <loc/private.h>
5601 struct loc_network {
5602 @@ -436,7 +437,7 @@ LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* othe
5606 -static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
5607 +LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
5608 // Families must match
5609 if (self->family != other->family)
5611 @@ -1106,207 +1107,3 @@ int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
5612 struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
5613 return loc_network_ref(node->network);
5618 -struct loc_network_list {
5619 - struct loc_ctx* ctx;
5622 - struct loc_network* list[1024];
5627 -LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
5628 - struct loc_network_list** list) {
5629 - struct loc_network_list* l = calloc(1, sizeof(*l));
5633 - l->ctx = loc_ref(ctx);
5636 - // Do not allow this list to grow larger than this
5637 - l->max_size = 1024;
5639 - DEBUG(l->ctx, "Network list allocated at %p\n", l);
5644 -LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
5650 -static void loc_network_list_free(struct loc_network_list* list) {
5651 - DEBUG(list->ctx, "Releasing network list at %p\n", list);
5653 - for (unsigned int i = 0; i < list->size; i++)
5654 - loc_network_unref(list->list[i]);
5656 - loc_unref(list->ctx);
5660 -LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
5664 - if (--list->refcount > 0)
5667 - loc_network_list_free(list);
5671 -LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
5672 - return list->size;
5675 -LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
5676 - return list->size == 0;
5679 -LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
5680 - for (unsigned int i = 0; i < list->size; i++)
5681 - loc_network_unref(list->list[i]);
5686 -LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
5687 - struct loc_network* network;
5690 - for (unsigned int i = 0; i < list->size; i++) {
5691 - network = list->list[i];
5693 - s = loc_network_str(network);
5695 - INFO(list->ctx, "%s\n", s);
5700 -LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
5702 - if (index >= list->size)
5705 - return loc_network_ref(list->list[index]);
5708 -LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
5709 - // Do not add networks that are already on the list
5710 - if (loc_network_list_contains(list, network))
5713 - // Check if we have space left
5714 - if (list->size == list->max_size) {
5715 - ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
5719 - DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
5721 - list->list[list->size++] = loc_network_ref(network);
5726 -LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
5727 - // Return nothing when empty
5728 - if (loc_network_list_empty(list)) {
5729 - DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5733 - struct loc_network* network = list->list[--list->size];
5735 - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5740 -LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
5741 - // Return nothing when empty
5742 - if (loc_network_list_empty(list)) {
5743 - DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5747 - struct loc_network* network = list->list[0];
5749 - // Move all elements to the top of the stack
5750 - for (unsigned int i = 0; i < --list->size; i++) {
5751 - list->list[i] = list->list[i+1];
5754 - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5759 -LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
5760 - for (unsigned int i = 0; i < list->size; i++) {
5761 - if (loc_network_eq(list->list[i], network))
5768 -static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
5769 - // Do nothing for invalid indices
5770 - if (i1 >= list->size || i2 >= list->size)
5773 - struct loc_network* network1 = list->list[i1];
5774 - struct loc_network* network2 = list->list[i2];
5776 - list->list[i1] = network2;
5777 - list->list[i2] = network1;
5780 -LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
5781 - unsigned int i = 0;
5782 - unsigned int j = list->size - 1;
5785 - loc_network_list_swap(list, i++, j--);
5789 -LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
5790 - unsigned int n = list->size;
5796 - for (unsigned int i = 1; i < n; i++) {
5797 - if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
5798 - loc_network_list_swap(list, i-1, i);
5804 - } while (swapped);
5807 -LOC_EXPORT int loc_network_list_merge(
5808 - struct loc_network_list* self, struct loc_network_list* other) {
5811 - for (unsigned int i = 0; i < other->size; i++) {
5812 - r = loc_network_list_push(self, other->list[i]);
5819 diff --git a/src/python/network.c b/src/python/network.c
5820 index 11f672b..ed91d65 100644
5821 --- a/src/python/network.c
5822 +++ b/src/python/network.c
5825 #include <loc/libloc.h>
5826 #include <loc/network.h>
5827 +#include <loc/network-list.h>
5829 #include "locationmodule.h"
5830 #include "network.h"
5834 From e646a8f35ec7eff009414b3fd107c9af5cf39a86 Mon Sep 17 00:00:00 2001
5835 From: Michael Tremer <michael.tremer@ipfire.org>
5836 Date: Mon, 16 Nov 2020 15:13:28 +0000
5837 Subject: [PATCH 56/70] Implement filtering for multiple countries in the
5840 This will allow us to speed up the export of the database
5841 if only a few countries should be returned.
5843 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
5846 src/country-list.c | 138 +++++++++++++++++++++++++++++++++++++++++
5848 src/database.c | 47 ++++++--------
5849 src/libloc.sym | 15 ++++-
5850 src/loc/country-list.h | 43 +++++++++++++
5851 src/loc/database.h | 5 +-
5852 src/python/database.c | 57 ++++++++++++++---
5853 8 files changed, 274 insertions(+), 36 deletions(-)
5854 create mode 100644 src/country-list.c
5855 create mode 100644 src/loc/country-list.h
5857 diff --git a/Makefile.am b/Makefile.am
5858 index f0d8c4c..f4ca3c8 100644
5861 @@ -93,6 +93,7 @@ pkginclude_HEADERS = \
5865 + src/loc/country-list.h \
5866 src/loc/database.h \
5869 @@ -109,6 +110,7 @@ src_libloc_la_SOURCES = \
5873 + src/country-list.c \
5876 src/network-list.c \
5877 diff --git a/src/country-list.c b/src/country-list.c
5878 new file mode 100644
5879 index 0000000..ae0d71a
5881 +++ b/src/country-list.c
5884 + libloc - A library to determine the location of someone on the Internet
5886 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
5888 + This library is free software; you can redistribute it and/or
5889 + modify it under the terms of the GNU Lesser General Public
5890 + License as published by the Free Software Foundation; either
5891 + version 2.1 of the License, or (at your option) any later version.
5893 + This library is distributed in the hope that it will be useful,
5894 + but WITHOUT ANY WARRANTY; without even the implied warranty of
5895 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5896 + Lesser General Public License for more details.
5900 +#include <stdlib.h>
5902 +#include <loc/country.h>
5903 +#include <loc/country-list.h>
5904 +#include <loc/private.h>
5906 +struct loc_country_list {
5907 + struct loc_ctx* ctx;
5910 + struct loc_country* list[1024];
5915 +LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
5916 + struct loc_country_list** list) {
5917 + struct loc_country_list* l = calloc(1, sizeof(*l));
5921 + l->ctx = loc_ref(ctx);
5924 + // Do not allow this list to grow larger than this
5925 + l->max_size = 1024;
5927 + DEBUG(l->ctx, "Country list allocated at %p\n", l);
5933 +LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
5939 +static void loc_country_list_free(struct loc_country_list* list) {
5940 + DEBUG(list->ctx, "Releasing country list at %p\n", list);
5942 + loc_country_list_clear(list);
5944 + loc_unref(list->ctx);
5948 +LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
5952 + if (--list->refcount > 0)
5955 + loc_country_list_free(list);
5959 +LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
5960 + return list->size;
5963 +LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
5964 + return list->size == 0;
5967 +LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
5968 + for (unsigned int i = 0; i < list->size; i++)
5969 + loc_country_unref(list->list[i]);
5972 +LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
5974 + if (index >= list->size)
5977 + return loc_country_ref(list->list[index]);
5980 +LOC_EXPORT int loc_country_list_append(
5981 + struct loc_country_list* list, struct loc_country* country) {
5982 + if (loc_country_list_contains(list, country))
5985 + // Check if we have space left
5986 + if (list->size == list->max_size) {
5987 + ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
5991 + DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
5993 + list->list[list->size++] = loc_country_ref(country);
5998 +LOC_EXPORT int loc_country_list_contains(
5999 + struct loc_country_list* list, struct loc_country* country) {
6000 + for (unsigned int i = 0; i < list->size; i++) {
6001 + if (loc_country_cmp(country, list->list[i]) == 0)
6008 +LOC_EXPORT int loc_country_list_contains_code(
6009 + struct loc_country_list* list, const char* code) {
6010 + struct loc_country* country;
6012 + int r = loc_country_new(list->ctx, &country, code);
6016 + r = loc_country_list_contains(list, country);
6017 + loc_country_unref(country);
6021 diff --git a/src/country.c b/src/country.c
6022 index 2ba93e6..7aac0db 100644
6025 @@ -34,6 +34,9 @@ struct loc_country {
6028 LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
6029 + if (!loc_country_code_is_valid(country_code))
6032 struct loc_country* c = calloc(1, sizeof(*c));
6035 diff --git a/src/database.c b/src/database.c
6036 index b9d870f..29823b2 100644
6037 --- a/src/database.c
6038 +++ b/src/database.c
6041 #include <loc/compat.h>
6042 #include <loc/country.h>
6043 +#include <loc/country-list.h>
6044 #include <loc/database.h>
6045 #include <loc/format.h>
6046 #include <loc/network.h>
6047 @@ -99,7 +100,7 @@ struct loc_database_enumerator {
6051 - char country_code[3];
6052 + struct loc_country_list* countries;
6054 enum loc_network_flags flags;
6056 @@ -1017,33 +1018,20 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
6060 -LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
6061 - // Set empty country code
6062 - if (!country_code || !*country_code) {
6063 - *enumerator->country_code = '\0';
6066 +LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
6067 + struct loc_database_enumerator* enumerator) {
6068 + if (!enumerator->countries)
6071 - // Treat A1, A2, A3 as special country codes,
6072 - // but perform search for flags instead
6073 - if (strcmp(country_code, "A1") == 0) {
6074 - return loc_database_enumerator_set_flag(enumerator,
6075 - LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
6076 - } else if (strcmp(country_code, "A2") == 0) {
6077 - return loc_database_enumerator_set_flag(enumerator,
6078 - LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
6079 - } else if (strcmp(country_code, "A3") == 0) {
6080 - return loc_database_enumerator_set_flag(enumerator,
6081 - LOC_NETWORK_FLAG_ANYCAST);
6083 + return loc_country_list_ref(enumerator->countries);
6086 - // Country codes must be two characters
6087 - if (!loc_country_code_is_valid(country_code))
6089 +LOC_EXPORT int loc_database_enumerator_set_countries(
6090 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
6091 + if (enumerator->countries)
6092 + loc_country_list_unref(enumerator->countries);
6094 - for (unsigned int i = 0; i < 3; i++) {
6095 - enumerator->country_code[i] = country_code[i];
6097 + enumerator->countries = loc_country_list_ref(countries);
6101 @@ -1129,6 +1117,12 @@ static int loc_database_enumerator_stack_push_node(
6105 +static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
6106 + const char* country_code = loc_network_get_country_code(network);
6108 + return loc_country_list_contains_code(countries, country_code);
6111 static int loc_database_enumerator_filter_network(
6112 struct loc_database_enumerator* enumerator, struct loc_network* network) {
6113 // Skip if the family does not match
6114 @@ -1136,8 +1130,7 @@ static int loc_database_enumerator_filter_network(
6117 // Skip if the country code does not match
6118 - if (*enumerator->country_code &&
6119 - !loc_network_match_country_code(network, enumerator->country_code))
6120 + if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
6123 // Skip if the ASN does not match
6124 diff --git a/src/libloc.sym b/src/libloc.sym
6125 index 453a1be..40e9f88 100644
6126 --- a/src/libloc.sym
6127 +++ b/src/libloc.sym
6128 @@ -49,6 +49,18 @@ global:
6129 loc_country_set_name;
6133 + loc_country_list_append;
6134 + loc_country_list_clear;
6135 + loc_country_list_contains;
6136 + loc_country_list_contains_code;
6137 + loc_country_list_empty;
6138 + loc_country_list_get;
6139 + loc_country_list_new;
6140 + loc_country_list_ref;
6141 + loc_country_list_size;
6142 + loc_country_list_unref;
6145 loc_database_add_as;
6146 loc_database_count_as;
6147 @@ -66,13 +78,14 @@ global:
6148 loc_database_verify;
6150 # Database Enumerator
6151 + loc_database_enumerator_get_countries;
6152 loc_database_enumerator_new;
6153 loc_database_enumerator_next_as;
6154 loc_database_enumerator_next_country;
6155 loc_database_enumerator_next_network;
6156 loc_database_enumerator_ref;
6157 loc_database_enumerator_set_asn;
6158 - loc_database_enumerator_set_country_code;
6159 + loc_database_enumerator_set_countries;
6160 loc_database_enumerator_set_family;
6161 loc_database_enumerator_set_flag;
6162 loc_database_enumerator_set_string;
6163 diff --git a/src/loc/country-list.h b/src/loc/country-list.h
6164 new file mode 100644
6165 index 0000000..a7f818a
6167 +++ b/src/loc/country-list.h
6170 + libloc - A library to determine the location of someone on the Internet
6172 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
6174 + This library is free software; you can redistribute it and/or
6175 + modify it under the terms of the GNU Lesser General Public
6176 + License as published by the Free Software Foundation; either
6177 + version 2.1 of the License, or (at your option) any later version.
6179 + This library is distributed in the hope that it will be useful,
6180 + but WITHOUT ANY WARRANTY; without even the implied warranty of
6181 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
6182 + Lesser General Public License for more details.
6185 +#ifndef LIBLOC_COUNTRY_LIST_H
6186 +#define LIBLOC_COUNTRY_LIST_H
6188 +#include <stdlib.h>
6190 +#include <loc/libloc.h>
6191 +#include <loc/country.h>
6193 +struct loc_country_list;
6195 +int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
6196 +struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
6197 +struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
6199 +size_t loc_country_list_size(struct loc_country_list* list);
6200 +int loc_country_list_empty(struct loc_country_list* list);
6201 +void loc_country_list_clear(struct loc_country_list* list);
6203 +struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
6204 +int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
6206 +int loc_country_list_contains(
6207 + struct loc_country_list* list, struct loc_country* country);
6208 +int loc_country_list_contains_code(
6209 + struct loc_country_list* list, const char* code);
6212 diff --git a/src/loc/database.h b/src/loc/database.h
6213 index 14eb5ea..246e5c5 100644
6214 --- a/src/loc/database.h
6215 +++ b/src/loc/database.h
6217 #include <loc/network.h>
6219 #include <loc/country.h>
6220 +#include <loc/country-list.h>
6222 struct loc_database;
6223 int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
6224 @@ -66,7 +67,9 @@ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_
6225 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
6227 int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
6228 -int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
6229 +struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
6230 +int loc_database_enumerator_set_countries(
6231 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
6232 int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
6233 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
6234 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
6235 diff --git a/src/python/database.c b/src/python/database.c
6236 index d169547..e6f6f37 100644
6237 --- a/src/python/database.c
6238 +++ b/src/python/database.c
6239 @@ -258,14 +258,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
6242 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
6243 - char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
6244 - const char* country_code = NULL;
6245 + char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
6246 + PyObject* country_codes = NULL;
6247 unsigned int asn = 0;
6252 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
6253 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
6254 + &PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
6257 struct loc_database_enumerator* enumerator;
6258 @@ -277,13 +278,55 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
6261 // Set country code we are searching for
6262 - if (country_code) {
6263 - r = loc_database_enumerator_set_country_code(enumerator, country_code);
6265 + if (country_codes) {
6266 + struct loc_country_list* countries;
6267 + r = loc_country_list_new(loc_ctx, &countries);
6269 - PyErr_SetFromErrno(PyExc_SystemError);
6270 + PyErr_SetString(PyExc_SystemError, "Could not create country list");
6274 + for (unsigned int i = 0; i < PyList_Size(country_codes); i++) {
6275 + PyObject* item = PyList_GetItem(country_codes, i);
6277 + if (!PyUnicode_Check(item)) {
6278 + PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
6279 + loc_country_list_unref(countries);
6283 + const char* country_code = PyUnicode_AsUTF8(item);
6285 + struct loc_country* country;
6286 + r = loc_country_new(loc_ctx, &country, country_code);
6288 + if (r == -EINVAL) {
6289 + PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
6291 + PyErr_SetString(PyExc_SystemError, "Could not create country");
6294 + loc_country_list_unref(countries);
6298 + // Append it to the list
6299 + r = loc_country_list_append(countries, country);
6301 + PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
6303 + loc_country_list_unref(countries);
6304 + loc_country_unref(country);
6308 + loc_country_unref(country);
6311 + loc_database_enumerator_set_countries(enumerator, countries);
6313 + Py_DECREF(country_codes);
6314 + loc_country_list_unref(countries);
6317 // Set the ASN we are searching for
6321 From 7af51f8a579c79714992a3e175036fb511139310 Mon Sep 17 00:00:00 2001
6322 From: Michael Tremer <michael.tremer@ipfire.org>
6323 Date: Mon, 16 Nov 2020 15:20:50 +0000
6324 Subject: [PATCH 57/70] python: Only return country codes we want
6326 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6328 src/python/export.py | 8 +++++++-
6329 1 file changed, 7 insertions(+), 1 deletion(-)
6331 diff --git a/src/python/export.py b/src/python/export.py
6332 index be4a68e..5e7fe53 100644
6333 --- a/src/python/export.py
6334 +++ b/src/python/export.py
6335 @@ -184,8 +184,14 @@ class Exporter(object):
6337 writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
6339 + # Filter countries from special country codes
6341 + country_code for country_code in countries if not country_code in flags.values()
6344 # Get all networks that match the family
6345 - networks = self.db.search_networks(family=family, flatten=True)
6346 + networks = self.db.search_networks(family=family,
6347 + country_codes=country_codes, flatten=True)
6349 # Walk through all networks
6350 for network in networks:
6354 From bd1dc6bf6fe4ce40bf12e7426e283b31afd274e1 Mon Sep 17 00:00:00 2001
6355 From: Michael Tremer <michael.tremer@ipfire.org>
6356 Date: Mon, 16 Nov 2020 15:25:15 +0000
6357 Subject: [PATCH 58/70] database: Filter flags in C
6359 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6361 src/python/export.py | 16 +++++++++++-----
6362 1 file changed, 11 insertions(+), 5 deletions(-)
6364 diff --git a/src/python/export.py b/src/python/export.py
6365 index 5e7fe53..739742f 100644
6366 --- a/src/python/export.py
6367 +++ b/src/python/export.py
6368 @@ -29,7 +29,7 @@ import _location
6369 log = logging.getLogger("location.export")
6374 _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
6375 _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
6376 _location.NETWORK_FLAG_ANYCAST : "A3",
6377 @@ -186,12 +186,18 @@ class Exporter(object):
6379 # Filter countries from special country codes
6381 - country_code for country_code in countries if not country_code in flags.values()
6382 + country_code for country_code in countries if not country_code in FLAGS.values()
6387 + for flag in FLAGS:
6388 + if FLAGS[flag] in countries:
6391 # Get all networks that match the family
6392 networks = self.db.search_networks(family=family,
6393 - country_codes=country_codes, flatten=True)
6394 + country_codes=country_codes, flags=flags, flatten=True)
6396 # Walk through all networks
6397 for network in networks:
6398 @@ -208,10 +214,10 @@ class Exporter(object):
6402 - for flag in flags:
6403 + for flag in FLAGS:
6404 if network.has_flag(flag):
6405 # Fetch the "fake" country code
6406 - country = flags[flag]
6407 + country = FLAGS[flag]
6410 writers[country].write(network)
6414 From 84a2f0c2d9cbf8ae4225802c29ccba86561c77ed Mon Sep 17 00:00:00 2001
6415 From: Michael Tremer <michael.tremer@ipfire.org>
6416 Date: Tue, 17 Nov 2020 16:46:48 +0000
6417 Subject: [PATCH 59/70] as: Add list for easier processing
6419 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6422 src/as-list.c | 138 ++++++++++++++++++++++++++++++++++++++++++
6423 src/database.c | 29 +++++++--
6424 src/libloc.sym | 15 ++++-
6425 src/loc/as-list.h | 41 +++++++++++++
6426 src/loc/database.h | 5 +-
6427 src/python/database.c | 58 ++++++++++++++++--
6428 src/python/export.py | 2 +-
6429 8 files changed, 275 insertions(+), 15 deletions(-)
6430 create mode 100644 src/as-list.c
6431 create mode 100644 src/loc/as-list.h
6433 diff --git a/Makefile.am b/Makefile.am
6434 index f4ca3c8..d0cc793 100644
6437 @@ -91,6 +91,7 @@ EXTRA_DIST += \
6438 pkginclude_HEADERS = \
6441 + src/loc/as-list.h \
6444 src/loc/country-list.h \
6445 @@ -109,6 +110,7 @@ lib_LTLIBRARIES = \
6446 src_libloc_la_SOURCES = \
6451 src/country-list.c \
6453 diff --git a/src/as-list.c b/src/as-list.c
6454 new file mode 100644
6455 index 0000000..7c69eb0
6460 + libloc - A library to determine the location of someone on the Internet
6462 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
6464 + This library is free software; you can redistribute it and/or
6465 + modify it under the terms of the GNU Lesser General Public
6466 + License as published by the Free Software Foundation; either
6467 + version 2.1 of the License, or (at your option) any later version.
6469 + This library is distributed in the hope that it will be useful,
6470 + but WITHOUT ANY WARRANTY; without even the implied warranty of
6471 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
6472 + Lesser General Public License for more details.
6476 +#include <stdlib.h>
6478 +#include <loc/as.h>
6479 +#include <loc/as-list.h>
6480 +#include <loc/private.h>
6482 +struct loc_as_list {
6483 + struct loc_ctx* ctx;
6486 + struct loc_as* list[1024];
6491 +LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
6492 + struct loc_as_list** list) {
6493 + struct loc_as_list* l = calloc(1, sizeof(*l));
6497 + l->ctx = loc_ref(ctx);
6500 + // Do not allow this list to grow larger than this
6501 + l->max_size = 1024;
6503 + DEBUG(l->ctx, "AS list allocated at %p\n", l);
6509 +LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
6515 +static void loc_as_list_free(struct loc_as_list* list) {
6516 + DEBUG(list->ctx, "Releasing AS list at %p\n", list);
6518 + loc_as_list_clear(list);
6520 + loc_unref(list->ctx);
6524 +LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
6528 + if (--list->refcount > 0)
6531 + loc_as_list_free(list);
6535 +LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
6536 + return list->size;
6539 +LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
6540 + return list->size == 0;
6543 +LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
6544 + for (unsigned int i = 0; i < list->size; i++)
6545 + loc_as_unref(list->list[i]);
6548 +LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
6550 + if (index >= list->size)
6553 + return loc_as_ref(list->list[index]);
6556 +LOC_EXPORT int loc_as_list_append(
6557 + struct loc_as_list* list, struct loc_as* as) {
6558 + if (loc_as_list_contains(list, as))
6561 + // Check if we have space left
6562 + if (list->size == list->max_size) {
6563 + ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
6567 + DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
6569 + list->list[list->size++] = loc_as_ref(as);
6574 +LOC_EXPORT int loc_as_list_contains(
6575 + struct loc_as_list* list, struct loc_as* as) {
6576 + for (unsigned int i = 0; i < list->size; i++) {
6577 + if (loc_as_cmp(as, list->list[i]) == 0)
6584 +LOC_EXPORT int loc_as_list_contains_number(
6585 + struct loc_as_list* list, uint32_t number) {
6586 + struct loc_as* as;
6588 + int r = loc_as_new(list->ctx, &as, number);
6592 + r = loc_as_list_contains(list, as);
6597 diff --git a/src/database.c b/src/database.c
6598 index 29823b2..51cb5cd 100644
6599 --- a/src/database.c
6600 +++ b/src/database.c
6603 #include <loc/libloc.h>
6605 +#include <loc/as-list.h>
6606 #include <loc/compat.h>
6607 #include <loc/country.h>
6608 #include <loc/country-list.h>
6609 @@ -101,7 +102,7 @@ struct loc_database_enumerator {
6612 struct loc_country_list* countries;
6614 + struct loc_as_list* asns;
6615 enum loc_network_flags flags;
6618 @@ -1036,9 +1037,20 @@ LOC_EXPORT int loc_database_enumerator_set_countries(
6622 -LOC_EXPORT int loc_database_enumerator_set_asn(
6623 - struct loc_database_enumerator* enumerator, unsigned int asn) {
6624 - enumerator->asn = asn;
6625 +LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
6626 + struct loc_database_enumerator* enumerator) {
6627 + if (!enumerator->asns)
6630 + return loc_as_list_ref(enumerator->asns);
6633 +LOC_EXPORT int loc_database_enumerator_set_asns(
6634 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
6635 + if (enumerator->asns)
6636 + loc_as_list_unref(enumerator->asns);
6638 + enumerator->asns = loc_as_list_ref(asns);
6642 @@ -1123,6 +1135,12 @@ static int loc_network_match_countries(struct loc_network* network, struct loc_c
6643 return loc_country_list_contains_code(countries, country_code);
6646 +static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
6647 + uint32_t asn = loc_network_get_asn(network);
6649 + return loc_as_list_contains_number(asns, asn);
6652 static int loc_database_enumerator_filter_network(
6653 struct loc_database_enumerator* enumerator, struct loc_network* network) {
6654 // Skip if the family does not match
6655 @@ -1134,8 +1152,7 @@ static int loc_database_enumerator_filter_network(
6658 // Skip if the ASN does not match
6659 - if (enumerator->asn &&
6660 - !loc_network_match_asn(network, enumerator->asn))
6661 + if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
6664 // Skip if flags do not match
6665 diff --git a/src/libloc.sym b/src/libloc.sym
6666 index 40e9f88..53273cd 100644
6667 --- a/src/libloc.sym
6668 +++ b/src/libloc.sym
6669 @@ -37,6 +37,18 @@ global:
6674 + loc_as_list_append;
6675 + loc_as_list_clear;
6676 + loc_as_list_contains;
6677 + loc_as_list_contains_number;
6678 + loc_as_list_empty;
6683 + loc_as_list_unref;
6687 loc_country_code_is_valid;
6688 @@ -78,13 +90,14 @@ global:
6689 loc_database_verify;
6691 # Database Enumerator
6692 + loc_database_enumerator_get_asns;
6693 loc_database_enumerator_get_countries;
6694 loc_database_enumerator_new;
6695 loc_database_enumerator_next_as;
6696 loc_database_enumerator_next_country;
6697 loc_database_enumerator_next_network;
6698 loc_database_enumerator_ref;
6699 - loc_database_enumerator_set_asn;
6700 + loc_database_enumerator_set_asns;
6701 loc_database_enumerator_set_countries;
6702 loc_database_enumerator_set_family;
6703 loc_database_enumerator_set_flag;
6704 diff --git a/src/loc/as-list.h b/src/loc/as-list.h
6705 new file mode 100644
6706 index 0000000..7b5c4e8
6708 +++ b/src/loc/as-list.h
6711 + libloc - A library to determine the location of someone on the Internet
6713 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
6715 + This library is free software; you can redistribute it and/or
6716 + modify it under the terms of the GNU Lesser General Public
6717 + License as published by the Free Software Foundation; either
6718 + version 2.1 of the License, or (at your option) any later version.
6720 + This library is distributed in the hope that it will be useful,
6721 + but WITHOUT ANY WARRANTY; without even the implied warranty of
6722 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
6723 + Lesser General Public License for more details.
6726 +#ifndef LIBLOC_AS_LIST_H
6727 +#define LIBLOC_AS_LIST_H
6729 +#include <loc/as.h>
6730 +#include <loc/libloc.h>
6732 +struct loc_as_list;
6734 +int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
6735 +struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
6736 +struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
6738 +size_t loc_as_list_size(struct loc_as_list* list);
6739 +int loc_as_list_empty(struct loc_as_list* list);
6740 +void loc_as_list_clear(struct loc_as_list* list);
6742 +struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
6743 +int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
6745 +int loc_as_list_contains(
6746 + struct loc_as_list* list, struct loc_as* as);
6747 +int loc_as_list_contains_number(
6748 + struct loc_as_list* list, uint32_t number);
6751 diff --git a/src/loc/database.h b/src/loc/database.h
6752 index 246e5c5..70801f0 100644
6753 --- a/src/loc/database.h
6754 +++ b/src/loc/database.h
6755 @@ -70,7 +70,10 @@ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerato
6756 struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
6757 int loc_database_enumerator_set_countries(
6758 struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
6759 -int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
6760 +struct loc_as_list* loc_database_enumerator_get_asns(
6761 + struct loc_database_enumerator* enumerator);
6762 +int loc_database_enumerator_set_asns(
6763 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
6764 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
6765 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
6766 int loc_database_enumerator_next_as(
6767 diff --git a/src/python/database.c b/src/python/database.c
6768 index e6f6f37..38a804c 100644
6769 --- a/src/python/database.c
6770 +++ b/src/python/database.c
6774 #include <loc/libloc.h>
6775 +#include <loc/as.h>
6776 +#include <loc/as-list.h>
6777 #include <loc/database.h>
6779 #include "locationmodule.h"
6780 @@ -258,15 +260,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
6783 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
6784 - char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
6785 + char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
6786 PyObject* country_codes = NULL;
6787 - unsigned int asn = 0;
6788 + PyObject* asn_list = NULL;
6793 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
6794 - &PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
6795 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
6796 + &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
6799 struct loc_database_enumerator* enumerator;
6800 @@ -330,13 +332,57 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
6803 // Set the ASN we are searching for
6805 - r = loc_database_enumerator_set_asn(enumerator, asn);
6807 + struct loc_as_list* asns;
6808 + r = loc_as_list_new(loc_ctx, &asns);
6810 + PyErr_SetString(PyExc_SystemError, "Could not create AS list");
6814 + for (unsigned int i = 0; i < PyList_Size(asn_list); i++) {
6815 + PyObject* item = PyList_GetItem(asn_list, i);
6817 + if (!PyLong_Check(item)) {
6818 + PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
6820 + loc_as_list_unref(asns);
6824 + unsigned long number = PyLong_AsLong(item);
6826 + struct loc_as* as;
6827 + r = loc_as_new(loc_ctx, &as, number);
6829 + PyErr_SetString(PyExc_SystemError, "Could not create AS");
6831 + loc_as_list_unref(asns);
6836 + r = loc_as_list_append(asns, as);
6838 + PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
6840 + loc_as_list_unref(asns);
6848 + r = loc_database_enumerator_set_asns(enumerator, asns);
6850 PyErr_SetFromErrno(PyExc_SystemError);
6852 + loc_as_list_unref(asns);
6856 + loc_as_list_unref(asns);
6859 // Set the flags we are searching for
6860 diff --git a/src/python/export.py b/src/python/export.py
6861 index 739742f..f675eb3 100644
6862 --- a/src/python/export.py
6863 +++ b/src/python/export.py
6864 @@ -197,7 +197,7 @@ class Exporter(object):
6866 # Get all networks that match the family
6867 networks = self.db.search_networks(family=family,
6868 - country_codes=country_codes, flags=flags, flatten=True)
6869 + country_codes=country_codes, asns=asns, flags=flags, flatten=True)
6871 # Walk through all networks
6872 for network in networks:
6876 From 50120b991fc2fa4b7813096de87b42d700faf3e6 Mon Sep 17 00:00:00 2001
6877 From: Michael Tremer <michael.tremer@ipfire.org>
6878 Date: Tue, 17 Nov 2020 16:56:43 +0000
6879 Subject: [PATCH 60/70] database: Simplify network matching code
6881 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6883 src/database.c | 16 ++++++++--------
6884 1 file changed, 8 insertions(+), 8 deletions(-)
6886 diff --git a/src/database.c b/src/database.c
6887 index 51cb5cd..1a354f6 100644
6888 --- a/src/database.c
6889 +++ b/src/database.c
6890 @@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
6894 -static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
6895 - const char* country_code = loc_network_get_country_code(network);
6897 - return loc_country_list_contains_code(countries, country_code);
6900 static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
6901 uint32_t asn = loc_network_get_asn(network);
6903 @@ -1148,8 +1142,14 @@ static int loc_database_enumerator_filter_network(
6906 // Skip if the country code does not match
6907 - if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
6909 + if (enumerator->countries) {
6910 + if (!loc_country_list_empty(enumerator->countries)) {
6911 + const char* country_code = loc_network_get_country_code(network);
6913 + if (!loc_country_list_contains_code(enumerator->countries, country_code))
6918 // Skip if the ASN does not match
6919 if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
6923 From c1a36c943181da5cd2aef589a972d5027e529eb8 Mon Sep 17 00:00:00 2001
6924 From: Michael Tremer <michael.tremer@ipfire.org>
6925 Date: Tue, 17 Nov 2020 16:58:55 +0000
6926 Subject: [PATCH 61/70] database: Simplify AS matching code
6928 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6930 src/database.c | 16 ++++++++--------
6931 1 file changed, 8 insertions(+), 8 deletions(-)
6933 diff --git a/src/database.c b/src/database.c
6934 index 1a354f6..be93e00 100644
6935 --- a/src/database.c
6936 +++ b/src/database.c
6937 @@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
6941 -static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
6942 - uint32_t asn = loc_network_get_asn(network);
6944 - return loc_as_list_contains_number(asns, asn);
6947 static int loc_database_enumerator_filter_network(
6948 struct loc_database_enumerator* enumerator, struct loc_network* network) {
6949 // Skip if the family does not match
6950 @@ -1152,8 +1146,14 @@ static int loc_database_enumerator_filter_network(
6953 // Skip if the ASN does not match
6954 - if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
6956 + if (enumerator->asns) {
6957 + if (!loc_as_list_empty(enumerator->asns)) {
6958 + uint32_t asn = loc_network_get_asn(network);
6960 + if (!loc_as_list_contains_number(enumerator->asns, asn))
6965 // Skip if flags do not match
6966 if (enumerator->flags &&
6970 From d5205091f9cc1ff987e483325d48696459df08d8 Mon Sep 17 00:00:00 2001
6971 From: Michael Tremer <michael.tremer@ipfire.org>
6972 Date: Tue, 17 Nov 2020 17:50:17 +0000
6973 Subject: [PATCH 62/70] countries: Make list grow dynamically
6975 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6977 src/country-list.c | 38 ++++++++++++++++++++++++++------------
6978 1 file changed, 26 insertions(+), 12 deletions(-)
6980 diff --git a/src/country-list.c b/src/country-list.c
6981 index ae0d71a..1ce2d06 100644
6982 --- a/src/country-list.c
6983 +++ b/src/country-list.c
6984 @@ -25,11 +25,27 @@ struct loc_country_list {
6985 struct loc_ctx* ctx;
6988 - struct loc_country* list[1024];
6989 + struct loc_country** elements;
6990 + size_t elements_size;
6996 +static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
6997 + DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
6998 + list, size, list->elements_size + size);
7000 + struct loc_country** elements = reallocarray(list->elements,
7001 + list->elements_size + size, sizeof(*list->elements));
7005 + list->elements = elements;
7006 + list->elements_size += size;
7011 LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
7012 struct loc_country_list** list) {
7013 struct loc_country_list* l = calloc(1, sizeof(*l));
7014 @@ -39,9 +55,6 @@ LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
7015 l->ctx = loc_ref(ctx);
7018 - // Do not allow this list to grow larger than this
7019 - l->max_size = 1024;
7021 DEBUG(l->ctx, "Country list allocated at %p\n", l);
7024 @@ -84,7 +97,7 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
7026 LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
7027 for (unsigned int i = 0; i < list->size; i++)
7028 - loc_country_unref(list->list[i]);
7029 + loc_country_unref(list->elements[i]);
7032 LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
7033 @@ -92,7 +105,7 @@ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* lis
7034 if (index >= list->size)
7037 - return loc_country_ref(list->list[index]);
7038 + return loc_country_ref(list->elements[index]);
7041 LOC_EXPORT int loc_country_list_append(
7042 @@ -101,14 +114,15 @@ LOC_EXPORT int loc_country_list_append(
7045 // Check if we have space left
7046 - if (list->size == list->max_size) {
7047 - ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
7049 + if (list->size >= list->elements_size) {
7050 + int r = loc_country_list_grow(list, 64);
7055 DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
7057 - list->list[list->size++] = loc_country_ref(country);
7058 + list->elements[list->size++] = loc_country_ref(country);
7062 @@ -116,7 +130,7 @@ LOC_EXPORT int loc_country_list_append(
7063 LOC_EXPORT int loc_country_list_contains(
7064 struct loc_country_list* list, struct loc_country* country) {
7065 for (unsigned int i = 0; i < list->size; i++) {
7066 - if (loc_country_cmp(country, list->list[i]) == 0)
7067 + if (loc_country_cmp(country, list->elements[i]) == 0)
7074 From 3b44e4211371d2103f89ba8f056b15edb7778fac Mon Sep 17 00:00:00 2001
7075 From: Michael Tremer <michael.tremer@ipfire.org>
7076 Date: Tue, 17 Nov 2020 17:55:51 +0000
7077 Subject: [PATCH 63/70] networks: Make list grow dynamically
7079 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7081 src/network-list.c | 60 ++++++++++++++++++++++++++++------------------
7082 1 file changed, 37 insertions(+), 23 deletions(-)
7084 diff --git a/src/network-list.c b/src/network-list.c
7085 index 1f6e80e..4912c02 100644
7086 --- a/src/network-list.c
7087 +++ b/src/network-list.c
7088 @@ -25,11 +25,27 @@ struct loc_network_list {
7089 struct loc_ctx* ctx;
7092 - struct loc_network* list[1024];
7093 + struct loc_network** elements;
7094 + size_t elements_size;
7100 +static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
7101 + DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
7102 + list, size, list->elements_size + size);
7104 + struct loc_network** elements = reallocarray(list->elements,
7105 + list->elements_size + size, sizeof(*list->elements));
7109 + list->elements = elements;
7110 + list->elements_size += size;
7115 LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
7116 struct loc_network_list** list) {
7117 struct loc_network_list* l = calloc(1, sizeof(*l));
7118 @@ -39,9 +55,6 @@ LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
7119 l->ctx = loc_ref(ctx);
7122 - // Do not allow this list to grow larger than this
7123 - l->max_size = 1024;
7125 DEBUG(l->ctx, "Network list allocated at %p\n", l);
7128 @@ -57,7 +70,7 @@ static void loc_network_list_free(struct loc_network_list* list) {
7129 DEBUG(list->ctx, "Releasing network list at %p\n", list);
7131 for (unsigned int i = 0; i < list->size; i++)
7132 - loc_network_unref(list->list[i]);
7133 + loc_network_unref(list->elements[i]);
7135 loc_unref(list->ctx);
7137 @@ -84,7 +97,7 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
7139 LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
7140 for (unsigned int i = 0; i < list->size; i++)
7141 - loc_network_unref(list->list[i]);
7142 + loc_network_unref(list->elements[i]);
7146 @@ -94,7 +107,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
7149 for (unsigned int i = 0; i < list->size; i++) {
7150 - network = list->list[i];
7151 + network = list->elements[i];
7153 s = loc_network_str(network);
7155 @@ -108,7 +121,7 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
7156 if (index >= list->size)
7159 - return loc_network_ref(list->list[index]);
7160 + return loc_network_ref(list->elements[index]);
7163 LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
7164 @@ -117,14 +130,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
7167 // Check if we have space left
7168 - if (list->size == list->max_size) {
7169 - ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
7171 + if (list->size >= list->elements_size) {
7172 + int r = loc_network_list_grow(list, 64);
7177 DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
7179 - list->list[list->size++] = loc_network_ref(network);
7180 + list->elements[list->size++] = loc_network_ref(network);
7184 @@ -136,7 +150,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
7188 - struct loc_network* network = list->list[--list->size];
7189 + struct loc_network* network = list->elements[--list->size];
7191 DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
7193 @@ -150,11 +164,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
7197 - struct loc_network* network = list->list[0];
7198 + struct loc_network* network = list->elements[0];
7200 // Move all elements to the top of the stack
7201 for (unsigned int i = 0; i < --list->size; i++) {
7202 - list->list[i] = list->list[i+1];
7203 + list->elements[i] = list->elements[i+1];
7206 DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
7207 @@ -164,7 +178,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
7209 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
7210 for (unsigned int i = 0; i < list->size; i++) {
7211 - if (loc_network_eq(list->list[i], network))
7212 + if (loc_network_eq(list->elements[i], network))
7216 @@ -176,11 +190,11 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
7217 if (i1 >= list->size || i2 >= list->size)
7220 - struct loc_network* network1 = list->list[i1];
7221 - struct loc_network* network2 = list->list[i2];
7222 + struct loc_network* network1 = list->elements[i1];
7223 + struct loc_network* network2 = list->elements[i2];
7225 - list->list[i1] = network2;
7226 - list->list[i2] = network1;
7227 + list->elements[i1] = network2;
7228 + list->elements[i2] = network1;
7231 LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
7232 @@ -200,7 +214,7 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
7235 for (unsigned int i = 1; i < n; i++) {
7236 - if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
7237 + if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) {
7238 loc_network_list_swap(list, i-1, i);
7241 @@ -215,7 +229,7 @@ LOC_EXPORT int loc_network_list_merge(
7244 for (unsigned int i = 0; i < other->size; i++) {
7245 - r = loc_network_list_push(self, other->list[i]);
7246 + r = loc_network_list_push(self, other->elements[i]);
7253 From 1a415f8c555f4fe9a68eb2a897c4a1fc0d33db25 Mon Sep 17 00:00:00 2001
7254 From: Michael Tremer <michael.tremer@ipfire.org>
7255 Date: Tue, 17 Nov 2020 17:57:55 +0000
7256 Subject: [PATCH 64/70] as: Make lists grow dynamically
7258 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7260 src/as-list.c | 38 ++++++++++++++++++++++++++------------
7261 1 file changed, 26 insertions(+), 12 deletions(-)
7263 diff --git a/src/as-list.c b/src/as-list.c
7264 index 7c69eb0..17de23e 100644
7267 @@ -25,11 +25,27 @@ struct loc_as_list {
7268 struct loc_ctx* ctx;
7271 - struct loc_as* list[1024];
7272 + struct loc_as** elements;
7273 + size_t elements_size;
7279 +static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
7280 + DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
7281 + list, size, list->elements_size + size);
7283 + struct loc_as** elements = reallocarray(list->elements,
7284 + list->elements_size + size, sizeof(*list->elements));
7288 + list->elements = elements;
7289 + list->elements_size += size;
7294 LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
7295 struct loc_as_list** list) {
7296 struct loc_as_list* l = calloc(1, sizeof(*l));
7297 @@ -39,9 +55,6 @@ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
7298 l->ctx = loc_ref(ctx);
7301 - // Do not allow this list to grow larger than this
7302 - l->max_size = 1024;
7304 DEBUG(l->ctx, "AS list allocated at %p\n", l);
7307 @@ -84,7 +97,7 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
7309 LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
7310 for (unsigned int i = 0; i < list->size; i++)
7311 - loc_as_unref(list->list[i]);
7312 + loc_as_unref(list->elements[i]);
7315 LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
7316 @@ -92,7 +105,7 @@ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index
7317 if (index >= list->size)
7320 - return loc_as_ref(list->list[index]);
7321 + return loc_as_ref(list->elements[index]);
7324 LOC_EXPORT int loc_as_list_append(
7325 @@ -101,14 +114,15 @@ LOC_EXPORT int loc_as_list_append(
7328 // Check if we have space left
7329 - if (list->size == list->max_size) {
7330 - ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
7332 + if (list->size >= list->elements_size) {
7333 + int r = loc_as_list_grow(list, 64);
7338 DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
7340 - list->list[list->size++] = loc_as_ref(as);
7341 + list->elements[list->size++] = loc_as_ref(as);
7345 @@ -116,7 +130,7 @@ LOC_EXPORT int loc_as_list_append(
7346 LOC_EXPORT int loc_as_list_contains(
7347 struct loc_as_list* list, struct loc_as* as) {
7348 for (unsigned int i = 0; i < list->size; i++) {
7349 - if (loc_as_cmp(as, list->list[i]) == 0)
7350 + if (loc_as_cmp(as, list->elements[i]) == 0)
7357 From e6592434ee7836507c1f436ec3b0db3bc81a81b9 Mon Sep 17 00:00:00 2001
7358 From: Michael Tremer <michael.tremer@ipfire.org>
7359 Date: Tue, 17 Nov 2020 18:13:49 +0000
7360 Subject: [PATCH 65/70] export: Change back to use Network objects
7362 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7364 src/python/export.py | 4 +---
7365 1 file changed, 1 insertion(+), 3 deletions(-)
7367 diff --git a/src/python/export.py b/src/python/export.py
7368 index f675eb3..67e437f 100644
7369 --- a/src/python/export.py
7370 +++ b/src/python/export.py
7371 @@ -144,9 +144,7 @@ class XTGeoIPOutputWriter(OutputWriter):
7372 def _write_network(self, network):
7373 for address in (network.first_address, network.last_address):
7374 # Convert this into a string of bits
7375 - bytes = socket.inet_pton(
7376 - socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
7378 + bytes = socket.inet_pton(network.family, address)
7385 From 248f5e0419f2349253b8ea96e477c15649fe2173 Mon Sep 17 00:00:00 2001
7386 From: Michael Tremer <michael.tremer@ipfire.org>
7387 Date: Tue, 17 Nov 2020 18:14:15 +0000
7388 Subject: [PATCH 66/70] Actually clear all lists
7390 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7392 src/as-list.c | 8 ++++++++
7393 src/country-list.c | 8 ++++++++
7394 src/network-list.c | 6 ++++++
7395 3 files changed, 22 insertions(+)
7397 diff --git a/src/as-list.c b/src/as-list.c
7398 index 17de23e..76620c7 100644
7401 @@ -96,8 +96,16 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
7404 LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
7405 + if (!list->elements)
7408 for (unsigned int i = 0; i < list->size; i++)
7409 loc_as_unref(list->elements[i]);
7411 + free(list->elements);
7412 + list->elements_size = 0;
7417 LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
7418 diff --git a/src/country-list.c b/src/country-list.c
7419 index 1ce2d06..1c49c47 100644
7420 --- a/src/country-list.c
7421 +++ b/src/country-list.c
7422 @@ -96,8 +96,16 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
7425 LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
7426 + if (!list->elements)
7429 for (unsigned int i = 0; i < list->size; i++)
7430 loc_country_unref(list->elements[i]);
7432 + free(list->elements);
7433 + list->elements_size = 0;
7438 LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
7439 diff --git a/src/network-list.c b/src/network-list.c
7440 index 4912c02..9cb4547 100644
7441 --- a/src/network-list.c
7442 +++ b/src/network-list.c
7443 @@ -96,9 +96,15 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
7446 LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
7447 + if (!list->elements)
7450 for (unsigned int i = 0; i < list->size; i++)
7451 loc_network_unref(list->elements[i]);
7453 + free(list->elements);
7454 + list->elements_size = 0;
7462 From c98ebf8aae2aa141193db52cd9429b1ded5b09c4 Mon Sep 17 00:00:00 2001
7463 From: Michael Tremer <michael.tremer@ipfire.org>
7464 Date: Tue, 17 Nov 2020 18:34:51 +0000
7465 Subject: [PATCH 67/70] database: Do not clean up python list
7467 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7469 src/python/database.c | 9 +++++++--
7470 1 file changed, 7 insertions(+), 2 deletions(-)
7472 diff --git a/src/python/database.c b/src/python/database.c
7473 index 38a804c..ed22275 100644
7474 --- a/src/python/database.c
7475 +++ b/src/python/database.c
7476 @@ -325,9 +325,14 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
7477 loc_country_unref(country);
7480 - loc_database_enumerator_set_countries(enumerator, countries);
7481 + r = loc_database_enumerator_set_countries(enumerator, countries);
7483 + PyErr_SetFromErrno(PyExc_SystemError);
7485 + loc_as_list_unref(countries);
7489 - Py_DECREF(country_codes);
7490 loc_country_list_unref(countries);
7496 From 5470d06cb59027f4e04b6d576763dbf7f1093fde Mon Sep 17 00:00:00 2001
7497 From: Michael Tremer <michael.tremer@ipfire.org>
7498 Date: Tue, 17 Nov 2020 19:01:04 +0000
7499 Subject: [PATCH 68/70] database: Free filter lists in enumerator
7501 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7503 src/database.c | 6 ++++++
7504 1 file changed, 6 insertions(+)
7506 diff --git a/src/database.c b/src/database.c
7507 index be93e00..ca35fe1 100644
7508 --- a/src/database.c
7509 +++ b/src/database.c
7510 @@ -950,6 +950,12 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera
7511 if (enumerator->string)
7512 free(enumerator->string);
7514 + if (enumerator->countries)
7515 + loc_country_list_unref(enumerator->countries);
7517 + if (enumerator->asns)
7518 + loc_as_list_unref(enumerator->asns);
7520 // Free network search
7521 free(enumerator->networks_visited);
7526 From e0e96878d3df51c4a265d51d088005dedf9335e3 Mon Sep 17 00:00:00 2001
7527 From: Michael Tremer <michael.tremer@ipfire.org>
7528 Date: Wed, 18 Nov 2020 13:18:52 +0000
7529 Subject: [PATCH 69/70] database: Add debug output to filtering
7531 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7533 src/database.c | 29 ++++++++++++++++-------------
7534 1 file changed, 16 insertions(+), 13 deletions(-)
7536 diff --git a/src/database.c b/src/database.c
7537 index ca35fe1..83dd752 100644
7538 --- a/src/database.c
7539 +++ b/src/database.c
7540 @@ -1138,33 +1138,36 @@ static int loc_database_enumerator_stack_push_node(
7541 static int loc_database_enumerator_filter_network(
7542 struct loc_database_enumerator* enumerator, struct loc_network* network) {
7543 // Skip if the family does not match
7544 - if (enumerator->family && loc_network_address_family(network) != enumerator->family)
7545 + if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
7546 + DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
7550 // Skip if the country code does not match
7551 - if (enumerator->countries) {
7552 - if (!loc_country_list_empty(enumerator->countries)) {
7553 - const char* country_code = loc_network_get_country_code(network);
7554 + if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
7555 + const char* country_code = loc_network_get_country_code(network);
7557 - if (!loc_country_list_contains_code(enumerator->countries, country_code))
7559 + if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
7560 + DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
7565 // Skip if the ASN does not match
7566 - if (enumerator->asns) {
7567 - if (!loc_as_list_empty(enumerator->asns)) {
7568 - uint32_t asn = loc_network_get_asn(network);
7569 + if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
7570 + uint32_t asn = loc_network_get_asn(network);
7572 - if (!loc_as_list_contains_number(enumerator->asns, asn))
7574 + if (!loc_as_list_contains_number(enumerator->asns, asn)) {
7575 + DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
7580 // Skip if flags do not match
7581 - if (enumerator->flags &&
7582 - !loc_network_match_flag(network, enumerator->flags))
7583 + if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
7584 + DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
7593 From bce0c9295ff8ff9488f24babe01ce851228d0b1e Mon Sep 17 00:00:00 2001
7594 From: Michael Tremer <michael.tremer@ipfire.org>
7595 Date: Wed, 18 Nov 2020 13:19:04 +0000
7596 Subject: [PATCH 70/70] export: Remove filtering for flags
7598 The filter is an AND filter and if we set the flags from
7599 the special country codes, we won't get back much.
7601 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7603 src/python/export.py | 8 +-------
7604 1 file changed, 1 insertion(+), 7 deletions(-)
7606 diff --git a/src/python/export.py b/src/python/export.py
7607 index 67e437f..4219957 100644
7608 --- a/src/python/export.py
7609 +++ b/src/python/export.py
7610 @@ -187,15 +187,9 @@ class Exporter(object):
7611 country_code for country_code in countries if not country_code in FLAGS.values()
7616 - for flag in FLAGS:
7617 - if FLAGS[flag] in countries:
7620 # Get all networks that match the family
7621 networks = self.db.search_networks(family=family,
7622 - country_codes=country_codes, asns=asns, flags=flags, flatten=True)
7623 + country_codes=country_codes, asns=asns, flatten=True)
7625 # Walk through all networks
7626 for network in networks: