1 From ee6ea3986dc80183157f67275dc9f28231b5d5b2 Mon Sep 17 00:00:00 2001
2 From: Michael Tremer <michael.tremer@ipfire.org>
3 Date: Thu, 24 Sep 2020 10:17:58 +0000
4 Subject: [PATCH 001/111] Revert "importer: Purge any redundant entries"
6 This reverts commit c2cc55d5a6875c3838f060032eaed89dcfb92ef6.
8 The query stalls the database and therefore the automatic
9 scripts are no longer able to generate a new version of the
12 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
14 src/python/location-importer.in | 22 +---------------------
15 1 file changed, 1 insertion(+), 21 deletions(-)
17 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
18 index 1467923..e3a07a0 100644
19 --- a/src/python/location-importer.in
20 +++ b/src/python/location-importer.in
21 @@ -374,27 +374,7 @@ class CLI(object):
22 INSERT INTO autnums(number, name)
23 SELECT _autnums.number, _organizations.name FROM _autnums
24 JOIN _organizations ON _autnums.organization = _organizations.handle
25 - ON CONFLICT (number) DO UPDATE SET name = excluded.name
29 - --- Purge any redundant entries
30 - CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
32 - SELECT network FROM networks candidates
34 - SELECT FROM networks
36 - networks.network << candidates.network
38 - networks.country = candidates.country
41 - CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
43 - DELETE FROM networks WHERE EXISTS (
44 - SELECT FROM _garbage WHERE networks.network = _garbage.network
46 + ON CONFLICT (number) DO UPDATE SET name = excluded.name;
49 # Download all extended sources
53 From 92f6abf4e272672bb0a71cfe991261b95ebe2fef Mon Sep 17 00:00:00 2001
54 From: Michael Tremer <michael.tremer@ipfire.org>
55 Date: Thu, 24 Sep 2020 10:18:58 +0000
56 Subject: [PATCH 002/111] Revert "importer: Import raw sources for inetnum's
59 This reverts commit 64e95fa903edec8b4e4e59830b395e2e4a411853.
61 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
63 src/python/importer.py | 14 ++++----
64 src/python/location-importer.in | 63 ---------------------------------
65 2 files changed, 7 insertions(+), 70 deletions(-)
67 diff --git a/src/python/importer.py b/src/python/importer.py
68 index f19db4b..de20f37 100644
69 --- a/src/python/importer.py
70 +++ b/src/python/importer.py
71 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
72 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
74 # Asia Pacific Network Information Centre
75 - "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
76 - "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
77 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
78 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
79 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
80 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
81 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
82 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
85 # Réseaux IP Européens
86 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
87 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
88 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
89 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
90 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
91 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
92 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
93 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
96 # African Network Information Centre
97 - #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
98 + "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
100 # Asia Pacific Network Information Centre
101 - #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
102 + "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
104 # American Registry for Internet Numbers
105 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
106 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
107 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
109 # Réseaux IP Européens
110 - #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
111 + "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
114 class Downloader(object):
115 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
116 index e3a07a0..77952f2 100644
117 --- a/src/python/location-importer.in
118 +++ b/src/python/location-importer.in
119 @@ -393,10 +393,6 @@ class CLI(object):
120 if line.startswith("aut-num:"):
121 return self._parse_autnum_block(block)
124 - if line.startswith("inet6num:") or line.startswith("inetnum:"):
125 - return self._parse_inetnum_block(block)
128 elif line.startswith("organisation:"):
129 return self._parse_org_block(block)
130 @@ -426,65 +422,6 @@ class CLI(object):
131 autnum.get("asn"), autnum.get("org"),
134 - def _parse_inetnum_block(self, block):
135 - logging.debug("Parsing inetnum block:")
139 - logging.debug(line)
142 - key, val = split_line(line)
144 - if key == "inetnum":
145 - start_address, delim, end_address = val.partition("-")
147 - # Strip any excess space
148 - start_address, end_address = start_address.rstrip(), end_address.strip()
150 - # Convert to IP address
152 - start_address = ipaddress.ip_address(start_address)
153 - end_address = ipaddress.ip_address(end_address)
155 - logging.warning("Could not parse line: %s" % line)
158 - # Set prefix to default
161 - # Count number of addresses in this subnet
162 - num_addresses = int(end_address) - int(start_address)
164 - prefix -= math.log(num_addresses, 2)
166 - inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
168 - elif key == "inet6num":
171 - elif key == "country":
172 - if val == "UNITED STATES":
175 - inetnum[key] = val.upper()
177 - # Skip empty objects
181 - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
183 - # Bail out in case we have processed a non-public IP network
184 - if network.is_private:
185 - logging.warning("Skipping non-globally routable network: %s" % network)
188 - self.db.execute("INSERT INTO networks(network, country) \
189 - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
190 - "%s" % network, inetnum.get("country"),
193 def _parse_org_block(self, block):
199 From f532841e9197ce2f40aad8c086d786b2cb783a54 Mon Sep 17 00:00:00 2001
200 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
201 Date: Mon, 12 Oct 2020 20:53:31 +0000
202 Subject: [PATCH 003/111] Revert "Revert "importer: Import raw sources for
205 This reverts commit 92f6abf4e272672bb0a71cfe991261b95ebe2fef.
207 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
209 src/python/importer.py | 14 ++++----
210 src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
211 2 files changed, 70 insertions(+), 7 deletions(-)
213 diff --git a/src/python/importer.py b/src/python/importer.py
214 index de20f37..f19db4b 100644
215 --- a/src/python/importer.py
216 +++ b/src/python/importer.py
217 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
218 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
220 # Asia Pacific Network Information Centre
221 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
222 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
223 + "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
224 + "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
225 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
226 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
227 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
228 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
231 # Réseaux IP Européens
232 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
233 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
234 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
235 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
236 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
237 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
238 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
239 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
242 # African Network Information Centre
243 - "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
244 + #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
246 # Asia Pacific Network Information Centre
247 - "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
248 + #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
250 # American Registry for Internet Numbers
251 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
252 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
253 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
255 # Réseaux IP Européens
256 - "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
257 + #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
260 class Downloader(object):
261 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
262 index 77952f2..e3a07a0 100644
263 --- a/src/python/location-importer.in
264 +++ b/src/python/location-importer.in
265 @@ -393,6 +393,10 @@ class CLI(object):
266 if line.startswith("aut-num:"):
267 return self._parse_autnum_block(block)
270 + if line.startswith("inet6num:") or line.startswith("inetnum:"):
271 + return self._parse_inetnum_block(block)
274 elif line.startswith("organisation:"):
275 return self._parse_org_block(block)
276 @@ -422,6 +426,65 @@ class CLI(object):
277 autnum.get("asn"), autnum.get("org"),
280 + def _parse_inetnum_block(self, block):
281 + logging.debug("Parsing inetnum block:")
285 + logging.debug(line)
288 + key, val = split_line(line)
290 + if key == "inetnum":
291 + start_address, delim, end_address = val.partition("-")
293 + # Strip any excess space
294 + start_address, end_address = start_address.rstrip(), end_address.strip()
296 + # Convert to IP address
298 + start_address = ipaddress.ip_address(start_address)
299 + end_address = ipaddress.ip_address(end_address)
301 + logging.warning("Could not parse line: %s" % line)
304 + # Set prefix to default
307 + # Count number of addresses in this subnet
308 + num_addresses = int(end_address) - int(start_address)
310 + prefix -= math.log(num_addresses, 2)
312 + inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
314 + elif key == "inet6num":
317 + elif key == "country":
318 + if val == "UNITED STATES":
321 + inetnum[key] = val.upper()
323 + # Skip empty objects
327 + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
329 + # Bail out in case we have processed a non-public IP network
330 + if network.is_private:
331 + logging.warning("Skipping non-globally routable network: %s" % network)
334 + self.db.execute("INSERT INTO networks(network, country) \
335 + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
336 + "%s" % network, inetnum.get("country"),
339 def _parse_org_block(self, block):
345 From a36bc686865fc87ea386fd90b389338bdcb80cbc Mon Sep 17 00:00:00 2001
346 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
347 Date: Mon, 12 Oct 2020 20:53:32 +0000
348 Subject: [PATCH 004/111] location-importer.in: only import relevant data from
349 AFRINIC, APNIC and RIPE
351 Content-Type: text/plain; charset=UTF-8
352 Content-Transfer-Encoding: 8bit
354 In contrast to ARIN and LACNIC, we are able to process more detailled
355 feeds from those RIRs, avoiding storage of obviously unnecessary data.
357 Thanks to various SQL optimisations, doing so now takes less time than
358 the first version of this did.
360 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
361 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
362 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
364 src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
365 1 file changed, 87 insertions(+), 2 deletions(-)
367 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
368 index e3a07a0..093f325 100644
369 --- a/src/python/location-importer.in
370 +++ b/src/python/location-importer.in
371 @@ -165,6 +165,7 @@ class CLI(object):
373 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
374 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
375 + CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
376 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
379 @@ -363,6 +364,16 @@ class CLI(object):
380 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
382 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
384 + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
386 + CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
387 + CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
390 + # Remove all previously imported content
391 + self.db.execute("""
392 + TRUNCATE TABLE networks;
395 for source in location.importer.WHOIS_SOURCES:
396 @@ -370,6 +381,67 @@ class CLI(object):
398 self._parse_block(block)
400 + # Process all parsed networks from every RIR we happen to have access to,
401 + # insert the largest network chunks into the networks table immediately...
402 + families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
404 + for family in (row.family for row in families):
405 + smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
407 + self.db.execute("INSERT INTO networks(network, country) \
408 + SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
410 + # ... determine any other prefixes for this network family, ...
411 + prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
412 + WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
414 + # ... and insert networks with this prefix in case they provide additional
415 + # information (i. e. subnet of a larger chunk with a different country)
416 + for prefix in (row.prefix for row in prefixes):
417 + self.db.execute("""
418 + WITH candidates AS (
425 + family(_rirdata.network) = %s
427 + masklen(_rirdata.network) = %s
431 + DISTINCT ON (c.network)
434 + masklen(networks.network),
435 + networks.country AS parent_country
441 + c.network << networks.network
444 + masklen(networks.network) DESC NULLS LAST
447 + networks(network, country)
454 + parent_country IS NULL
456 + country <> parent_country
457 + ON CONFLICT DO NOTHING""",
462 INSERT INTO autnums(number, name)
463 SELECT _autnums.number, _organizations.name FROM _autnums
464 @@ -470,17 +542,30 @@ class CLI(object):
465 inetnum[key] = val.upper()
469 + if not inetnum or not "country" in inetnum:
472 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
474 + # Bail out in case we have processed a network covering the entire IP range, which
475 + # is necessary to work around faulty (?) IPv6 network processing
476 + if network.prefixlen == 0:
477 + logging.warning("Skipping network covering the entire IP adress range: %s" % network)
480 + # Bail out in case we have processed a network whose prefix length indicates it is
481 + # not globally routable (we have decided not to process them at the moment, as they
482 + # significantly enlarge our database without providing very helpful additional information)
483 + if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
484 + logging.info("Skipping network too small to be publicly announced: %s" % network)
487 # Bail out in case we have processed a non-public IP network
488 if network.is_private:
489 logging.warning("Skipping non-globally routable network: %s" % network)
492 - self.db.execute("INSERT INTO networks(network, country) \
493 + self.db.execute("INSERT INTO _rirdata(network, country) \
494 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
495 "%s" % network, inetnum.get("country"),
500 From 2373de384f10f5573bbd7570f5522545df70c0e3 Mon Sep 17 00:00:00 2001
501 From: Michael Tremer <michael.tremer@ipfire.org>
502 Date: Fri, 16 Oct 2020 12:24:58 +0000
503 Subject: [PATCH 005/111] location-importer: Include all overridden networks
505 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
507 src/python/location-importer.in | 2 ++
508 1 file changed, 2 insertions(+)
510 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
511 index 093f325..d249a35 100644
512 --- a/src/python/location-importer.in
513 +++ b/src/python/location-importer.in
514 @@ -240,6 +240,8 @@ class CLI(object):
515 SELECT network FROM announcements
517 SELECT network FROM networks
519 + SELECT network FROM network_overrides
526 From 13f67f285856e8eabfeff2daf1be3aeaa36a82cc Mon Sep 17 00:00:00 2001
527 From: Michael Tremer <michael.tremer@ipfire.org>
528 Date: Fri, 16 Oct 2020 12:26:38 +0000
529 Subject: [PATCH 006/111] Revert "location-importer.in: only import relevant
530 data from AFRINIC, APNIC and RIPE"
532 This reverts commit a36bc686865fc87ea386fd90b389338bdcb80cbc.
534 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
536 src/python/location-importer.in | 89 +--------------------------------
537 1 file changed, 2 insertions(+), 87 deletions(-)
539 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
540 index d249a35..b220eaf 100644
541 --- a/src/python/location-importer.in
542 +++ b/src/python/location-importer.in
543 @@ -165,7 +165,6 @@ class CLI(object):
545 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
546 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
547 - CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
548 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
551 @@ -366,16 +365,6 @@ class CLI(object):
552 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
554 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
556 - CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
558 - CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
559 - CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
562 - # Remove all previously imported content
563 - self.db.execute("""
564 - TRUNCATE TABLE networks;
567 for source in location.importer.WHOIS_SOURCES:
568 @@ -383,67 +372,6 @@ class CLI(object):
570 self._parse_block(block)
572 - # Process all parsed networks from every RIR we happen to have access to,
573 - # insert the largest network chunks into the networks table immediately...
574 - families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
576 - for family in (row.family for row in families):
577 - smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
579 - self.db.execute("INSERT INTO networks(network, country) \
580 - SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
582 - # ... determine any other prefixes for this network family, ...
583 - prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
584 - WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
586 - # ... and insert networks with this prefix in case they provide additional
587 - # information (i. e. subnet of a larger chunk with a different country)
588 - for prefix in (row.prefix for row in prefixes):
589 - self.db.execute("""
590 - WITH candidates AS (
597 - family(_rirdata.network) = %s
599 - masklen(_rirdata.network) = %s
603 - DISTINCT ON (c.network)
606 - masklen(networks.network),
607 - networks.country AS parent_country
613 - c.network << networks.network
616 - masklen(networks.network) DESC NULLS LAST
619 - networks(network, country)
626 - parent_country IS NULL
628 - country <> parent_country
629 - ON CONFLICT DO NOTHING""",
634 INSERT INTO autnums(number, name)
635 SELECT _autnums.number, _organizations.name FROM _autnums
636 @@ -544,30 +472,17 @@ class CLI(object):
637 inetnum[key] = val.upper()
640 - if not inetnum or not "country" in inetnum:
644 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
646 - # Bail out in case we have processed a network covering the entire IP range, which
647 - # is necessary to work around faulty (?) IPv6 network processing
648 - if network.prefixlen == 0:
649 - logging.warning("Skipping network covering the entire IP adress range: %s" % network)
652 - # Bail out in case we have processed a network whose prefix length indicates it is
653 - # not globally routable (we have decided not to process them at the moment, as they
654 - # significantly enlarge our database without providing very helpful additional information)
655 - if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
656 - logging.info("Skipping network too small to be publicly announced: %s" % network)
659 # Bail out in case we have processed a non-public IP network
660 if network.is_private:
661 logging.warning("Skipping non-globally routable network: %s" % network)
664 - self.db.execute("INSERT INTO _rirdata(network, country) \
665 + self.db.execute("INSERT INTO networks(network, country) \
666 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
667 "%s" % network, inetnum.get("country"),
672 From 44341478233115b26bb27fdb24da5b0a1eedb173 Mon Sep 17 00:00:00 2001
673 From: Michael Tremer <michael.tremer@ipfire.org>
674 Date: Fri, 16 Oct 2020 12:26:43 +0000
675 Subject: [PATCH 007/111] Revert "Revert "Revert "importer: Import raw sources
676 for inetnum's again"""
678 This reverts commit f532841e9197ce2f40aad8c086d786b2cb783a54.
680 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
682 src/python/importer.py | 14 ++++----
683 src/python/location-importer.in | 63 ---------------------------------
684 2 files changed, 7 insertions(+), 70 deletions(-)
686 diff --git a/src/python/importer.py b/src/python/importer.py
687 index f19db4b..de20f37 100644
688 --- a/src/python/importer.py
689 +++ b/src/python/importer.py
690 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
691 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
693 # Asia Pacific Network Information Centre
694 - "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
695 - "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
696 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
697 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
698 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
699 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
700 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
701 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
704 # Réseaux IP Européens
705 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
706 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
707 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
708 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
709 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
710 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
711 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
712 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
715 # African Network Information Centre
716 - #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
717 + "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
719 # Asia Pacific Network Information Centre
720 - #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
721 + "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
723 # American Registry for Internet Numbers
724 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
725 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
726 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
728 # Réseaux IP Européens
729 - #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
730 + "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
733 class Downloader(object):
734 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
735 index b220eaf..e87d378 100644
736 --- a/src/python/location-importer.in
737 +++ b/src/python/location-importer.in
738 @@ -395,10 +395,6 @@ class CLI(object):
739 if line.startswith("aut-num:"):
740 return self._parse_autnum_block(block)
743 - if line.startswith("inet6num:") or line.startswith("inetnum:"):
744 - return self._parse_inetnum_block(block)
747 elif line.startswith("organisation:"):
748 return self._parse_org_block(block)
749 @@ -428,65 +424,6 @@ class CLI(object):
750 autnum.get("asn"), autnum.get("org"),
753 - def _parse_inetnum_block(self, block):
754 - logging.debug("Parsing inetnum block:")
758 - logging.debug(line)
761 - key, val = split_line(line)
763 - if key == "inetnum":
764 - start_address, delim, end_address = val.partition("-")
766 - # Strip any excess space
767 - start_address, end_address = start_address.rstrip(), end_address.strip()
769 - # Convert to IP address
771 - start_address = ipaddress.ip_address(start_address)
772 - end_address = ipaddress.ip_address(end_address)
774 - logging.warning("Could not parse line: %s" % line)
777 - # Set prefix to default
780 - # Count number of addresses in this subnet
781 - num_addresses = int(end_address) - int(start_address)
783 - prefix -= math.log(num_addresses, 2)
785 - inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
787 - elif key == "inet6num":
790 - elif key == "country":
791 - if val == "UNITED STATES":
794 - inetnum[key] = val.upper()
796 - # Skip empty objects
800 - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
802 - # Bail out in case we have processed a non-public IP network
803 - if network.is_private:
804 - logging.warning("Skipping non-globally routable network: %s" % network)
807 - self.db.execute("INSERT INTO networks(network, country) \
808 - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
809 - "%s" % network, inetnum.get("country"),
812 def _parse_org_block(self, block):
818 From a7d3a7a0565a0e09d3442e5829a0f30f016993b9 Mon Sep 17 00:00:00 2001
819 From: Michael Tremer <michael.tremer@ipfire.org>
820 Date: Tue, 20 Oct 2020 20:44:43 +0000
821 Subject: [PATCH 008/111] as: Fix dereferencing NULL pointer when setting AS
824 Reported-by: Gisle Vanem <gisle.vanem@gmail.com>
825 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
827 src/as.c | 8 +++++++-
828 1 file changed, 7 insertions(+), 1 deletion(-)
830 diff --git a/src/as.c b/src/as.c
831 index e1fbb01..8421ac8 100644
834 @@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
837 LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) {
838 - as->name = strdup(name);
843 + as->name = strdup(name);
852 From ddb326ad38a7c7202315dd2c6f938313db04ee22 Mon Sep 17 00:00:00 2001
853 From: Michael Tremer <michael.tremer@ipfire.org>
854 Date: Wed, 21 Oct 2020 09:18:08 +0000
855 Subject: [PATCH 009/111] as: Do not attempt to match name when it wasn't set
857 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
860 1 file changed, 4 insertions(+)
862 diff --git a/src/as.c b/src/as.c
863 index 8421ac8..757bf3d 100644
866 @@ -145,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
870 + // Cannot match anything when name is not set
874 // Search if string is in name
875 if (strcasestr(as->name, string) != NULL)
880 From d226ad2d97cbcd42ce807d9308569b1b9c5d4e2f Mon Sep 17 00:00:00 2001
881 From: Michael Tremer <michael.tremer@ipfire.org>
882 Date: Wed, 21 Oct 2020 09:28:39 +0000
883 Subject: [PATCH 010/111] writer: Free array with pointer to ASes, too
885 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
887 src/writer.c | 7 +++++--
888 1 file changed, 5 insertions(+), 2 deletions(-)
890 diff --git a/src/writer.c b/src/writer.c
891 index 5939cff..160650f 100644
894 @@ -147,8 +147,11 @@ static void loc_writer_free(struct loc_writer* writer) {
895 EVP_PKEY_free(writer->private_key2);
898 - for (unsigned int i = 0; i < writer->as_count; i++) {
899 - loc_as_unref(writer->as[i]);
901 + for (unsigned int i = 0; i < writer->as_count; i++) {
902 + loc_as_unref(writer->as[i]);
907 // Release network tree
911 From d89a7d62772048ae1bd18d03f69df46b7e1a3d3c Mon Sep 17 00:00:00 2001
912 From: Michael Tremer <michael.tremer@ipfire.org>
913 Date: Wed, 21 Oct 2020 09:31:29 +0000
914 Subject: [PATCH 011/111] writer: Free countries when the writer is being
917 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
919 src/writer.c | 8 ++++++++
920 1 file changed, 8 insertions(+)
922 diff --git a/src/writer.c b/src/writer.c
923 index 160650f..2f09b56 100644
926 @@ -154,6 +154,14 @@ static void loc_writer_free(struct loc_writer* writer) {
930 + // Unref all countries
931 + if (writer->countries) {
932 + for (unsigned int i = 0; i < writer->countries_count; i++) {
933 + loc_country_unref(writer->countries[i]);
935 + free(writer->countries);
938 // Release network tree
939 if (writer->networks)
940 loc_network_tree_unref(writer->networks);
944 From 0f1aedbc68e3945770c93e0ebd83eed0f555d6f0 Mon Sep 17 00:00:00 2001
945 From: Michael Tremer <michael.tremer@ipfire.org>
946 Date: Wed, 21 Oct 2020 13:19:44 +0000
947 Subject: [PATCH 012/111] tests: Try adding an invalid network
949 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
951 src/test-network.c | 8 ++++++++
952 1 file changed, 8 insertions(+)
954 diff --git a/src/test-network.c b/src/test-network.c
955 index d38f13d..e908b57 100644
956 --- a/src/test-network.c
957 +++ b/src/test-network.c
958 @@ -160,6 +160,14 @@ int main(int argc, char** argv) {
960 loc_network_set_asn(network4, 1024);
962 + // Try adding an invalid network
963 + struct loc_network* network;
964 + err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
965 + if (err != -EINVAL) {
966 + fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
967 + exit(EXIT_FAILURE);
972 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
976 From 13ad6e695f9ffc7847b3afe3e9cbcea8fb3a443f Mon Sep 17 00:00:00 2001
977 From: Michael Tremer <michael.tremer@ipfire.org>
978 Date: Wed, 21 Oct 2020 13:36:35 +0000
979 Subject: [PATCH 013/111] networks: Improve parsing IP addresses
981 loc_network_new_from_string() seem to have had some unexpected
982 behaviour for invalid inputs.
984 The function has been tidied up slightly and returns as soon as
985 some invalid input was detected.
987 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
989 src/network.c | 52 +++++++++++++++++++++++++++++++---------------
990 src/test-network.c | 7 +++++++
991 2 files changed, 42 insertions(+), 17 deletions(-)
993 diff --git a/src/network.c b/src/network.c
994 index 366caa2..c112a41 100644
997 @@ -160,9 +160,10 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
998 LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
999 const char* address_string) {
1000 struct in6_addr first_address;
1001 - unsigned int prefix = 0;
1002 char* prefix_string;
1006 + DEBUG(ctx, "Attempting to parse network %s\n", address_string);
1008 // Make a copy of the string to work on it
1009 char* buffer = strdup(address_string);
1010 @@ -171,29 +172,46 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1011 // Split address and prefix
1012 address_string = strsep(&prefix_string, "/");
1014 - // Did we find a prefix?
1015 - if (prefix_string) {
1016 - // Convert prefix to integer
1017 - prefix = strtol(prefix_string, NULL, 10);
1018 + DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
1021 - // Parse the address
1022 - r = loc_parse_address(ctx, address_string, &first_address);
1023 + // We need to have a prefix
1024 + if (!prefix_string) {
1025 + DEBUG(ctx, "No prefix set\n");
1029 - // Map the prefix to IPv6 if needed
1030 - if (IN6_IS_ADDR_V4MAPPED(&first_address))
1033 + // Convert prefix to integer
1034 + unsigned int prefix = strtol(prefix_string, NULL, 10);
1036 + // End if the prefix was invalid
1038 + DEBUG(ctx, "The prefix is zero or not a number\n");
1042 + // Parse the address
1043 + r = loc_parse_address(ctx, address_string, &first_address);
1045 + DEBUG(ctx, "The address could not be parsed\n");
1049 + // Map the prefix to IPv6 if needed
1050 + if (IN6_IS_ADDR_V4MAPPED(&first_address))
1054 // Free temporary buffer
1058 - r = loc_network_new(ctx, network, &first_address, prefix);
1060 + // Exit if the parsing was unsuccessful
1064 + DEBUG(ctx, "GOT HERE\n");
1067 + // Create a new network
1068 + return loc_network_new(ctx, network, &first_address, prefix);
1071 LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
1072 diff --git a/src/test-network.c b/src/test-network.c
1073 index e908b57..85eca00 100644
1074 --- a/src/test-network.c
1075 +++ b/src/test-network.c
1076 @@ -168,6 +168,13 @@ int main(int argc, char** argv) {
1080 + // Try adding a single address
1081 + err = loc_writer_add_network(writer, &network, "2001:db8::");
1082 + if (err != -EINVAL) {
1083 + fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
1084 + exit(EXIT_FAILURE);
1087 FILE* f = tmpfile();
1089 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
1093 From 6a467e9345bb5a3d37911c9aaac30019eaa4492b Mon Sep 17 00:00:00 2001
1094 From: Michael Tremer <michael.tremer@ipfire.org>
1095 Date: Wed, 21 Oct 2020 13:43:21 +0000
1096 Subject: [PATCH 014/111] networks: Test if we can add localhost (IPv6)
1098 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1100 src/test-network.c | 7 +++++++
1101 1 file changed, 7 insertions(+)
1103 diff --git a/src/test-network.c b/src/test-network.c
1104 index 85eca00..8c7e898 100644
1105 --- a/src/test-network.c
1106 +++ b/src/test-network.c
1107 @@ -175,6 +175,13 @@ int main(int argc, char** argv) {
1111 + // Try adding localhost
1112 + err = loc_writer_add_network(writer, &network, "::1/128");
1113 + if (err != -EINVAL) {
1114 + fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
1115 + exit(EXIT_FAILURE);
1118 FILE* f = tmpfile();
1120 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
1124 From fc1190aa11e3ff3d2dbf5f4d408c298e7916f46f Mon Sep 17 00:00:00 2001
1125 From: Michael Tremer <michael.tremer@ipfire.org>
1126 Date: Wed, 21 Oct 2020 13:44:50 +0000
1127 Subject: [PATCH 015/111] networks: Remove accidentially committed debug line
1129 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1131 src/network.c | 2 --
1132 1 file changed, 2 deletions(-)
1134 diff --git a/src/network.c b/src/network.c
1135 index c112a41..be88d75 100644
1138 @@ -208,8 +208,6 @@ FAIL:
1142 - DEBUG(ctx, "GOT HERE\n");
1144 // Create a new network
1145 return loc_network_new(ctx, network, &first_address, prefix);
1150 From a1707d8983898b6878cdd5c68744bcc444e278ed Mon Sep 17 00:00:00 2001
1151 From: Michael Tremer <michael.tremer@ipfire.org>
1152 Date: Wed, 21 Oct 2020 13:53:36 +0000
1153 Subject: [PATCH 016/111] importer: Add search index to announcements table
1155 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1157 src/python/location-importer.in | 1 +
1158 1 file changed, 1 insertion(+)
1160 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1161 index e87d378..d0fe5a6 100644
1162 --- a/src/python/location-importer.in
1163 +++ b/src/python/location-importer.in
1164 @@ -152,6 +152,7 @@ class CLI(object):
1165 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
1166 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
1167 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
1168 + CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
1171 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
1175 From 991baf530d47adb2ed7a15b65dc4565d07fa6d07 Mon Sep 17 00:00:00 2001
1176 From: Michael Tremer <michael.tremer@ipfire.org>
1177 Date: Wed, 21 Oct 2020 13:54:45 +0000
1178 Subject: [PATCH 017/111] importer: Add search index to network_overrides table
1180 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1182 src/python/location-importer.in | 2 ++
1183 1 file changed, 2 insertions(+)
1185 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1186 index d0fe5a6..fe21d73 100644
1187 --- a/src/python/location-importer.in
1188 +++ b/src/python/location-importer.in
1189 @@ -189,6 +189,8 @@ class CLI(object):
1191 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
1192 ON network_overrides(network);
1193 + CREATE INDEX IF NOT EXISTS network_overrides_search
1194 + ON network_overrides USING GIST(network inet_ops);
1201 From bbea93a74651df10e2ffdbd09eb434dc6a0471bc Mon Sep 17 00:00:00 2001
1202 From: Michael Tremer <michael.tremer@ipfire.org>
1203 Date: Wed, 21 Oct 2020 16:01:57 +0000
1204 Subject: [PATCH 018/111] importer: Restructure SQL query to be executed in
1207 There are no functional changes, this just runs quicker now.
1209 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1211 src/python/location-importer.in | 67 ++++++++++++++++++---------------
1212 1 file changed, 37 insertions(+), 30 deletions(-)
1214 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1215 index fe21d73..3c1e5e2 100644
1216 --- a/src/python/location-importer.in
1217 +++ b/src/python/location-importer.in
1218 @@ -237,34 +237,24 @@ class CLI(object):
1220 # Select all known networks
1221 rows = self.db.query("""
1222 - -- Get a (sorted) list of all known networks
1223 - WITH known_networks AS (
1224 - SELECT network FROM announcements
1226 - SELECT network FROM networks
1228 - SELECT network FROM network_overrides
1232 -- Return a list of those networks enriched with all
1233 -- other information that we store in the database
1235 - DISTINCT ON (known_networks.network)
1236 - known_networks.network AS network,
1237 - announcements.autnum AS autnum,
1238 + DISTINCT ON (network)
1245 SELECT country FROM network_overrides overrides
1246 - WHERE announcements.network <<= overrides.network
1247 + WHERE networks.network <<= overrides.network
1248 ORDER BY masklen(overrides.network) DESC
1252 SELECT country FROM autnum_overrides overrides
1253 - WHERE announcements.autnum = overrides.number
1254 + WHERE networks.autnum = overrides.number
1258 @@ -273,50 +263,67 @@ class CLI(object):
1261 SELECT is_anonymous_proxy FROM network_overrides overrides
1262 - WHERE announcements.network <<= overrides.network
1263 + WHERE networks.network <<= overrides.network
1264 ORDER BY masklen(overrides.network) DESC
1268 SELECT is_anonymous_proxy FROM autnum_overrides overrides
1269 - WHERE announcements.autnum = overrides.number
1270 + WHERE networks.autnum = overrides.number
1273 ) AS is_anonymous_proxy,
1276 SELECT is_satellite_provider FROM network_overrides overrides
1277 - WHERE announcements.network <<= overrides.network
1278 + WHERE networks.network <<= overrides.network
1279 ORDER BY masklen(overrides.network) DESC
1283 SELECT is_satellite_provider FROM autnum_overrides overrides
1284 - WHERE announcements.autnum = overrides.number
1285 + WHERE networks.autnum = overrides.number
1288 ) AS is_satellite_provider,
1291 SELECT is_anycast FROM network_overrides overrides
1292 - WHERE announcements.network <<= overrides.network
1293 + WHERE networks.network <<= overrides.network
1294 ORDER BY masklen(overrides.network) DESC
1298 SELECT is_anycast FROM autnum_overrides overrides
1299 - WHERE announcements.autnum = overrides.number
1300 + WHERE networks.autnum = overrides.number
1305 - -- Must be part of returned values for ORDER BY clause
1306 - masklen(announcements.network) AS sort_a,
1307 - masklen(networks.network) AS sort_b
1308 - FROM known_networks
1309 - LEFT JOIN announcements ON known_networks.network <<= announcements.network
1310 - LEFT JOIN networks ON known_networks.network <<= networks.network
1311 - ORDER BY known_networks.network, sort_a DESC, sort_b DESC
1315 + known_networks.network AS network,
1316 + announcements.autnum AS autnum,
1317 + networks.country AS country,
1319 + -- Must be part of returned values for ORDER BY clause
1320 + masklen(announcements.network) AS sort_a,
1321 + masklen(networks.network) AS sort_b
1323 + SELECT network FROM announcements
1325 + SELECT network FROM networks
1327 + SELECT network FROM network_overrides
1330 + announcements ON known_networks.network <<= announcements.network
1332 + networks ON known_networks.network <<= networks.network
1334 + known_networks.network,
1344 From 26ab419b68d166f932db1f97c38cb9d793d04187 Mon Sep 17 00:00:00 2001
1345 From: Michael Tremer <michael.tremer@ipfire.org>
1346 Date: Thu, 22 Oct 2020 12:24:34 +0000
1347 Subject: [PATCH 019/111] network: Allow adding single IP addresses and
1348 automatically add the prefix
1350 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1352 src/network.c | 33 +++++++++++++++------------------
1353 src/test-network.c | 4 ++--
1354 2 files changed, 17 insertions(+), 20 deletions(-)
1356 diff --git a/src/network.c b/src/network.c
1357 index be88d75..d7b1645 100644
1360 @@ -161,6 +161,7 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1361 const char* address_string) {
1362 struct in6_addr first_address;
1363 char* prefix_string;
1364 + unsigned int prefix = 128;
1367 DEBUG(ctx, "Attempting to parse network %s\n", address_string);
1368 @@ -174,21 +175,6 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1370 DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
1372 - // We need to have a prefix
1373 - if (!prefix_string) {
1374 - DEBUG(ctx, "No prefix set\n");
1378 - // Convert prefix to integer
1379 - unsigned int prefix = strtol(prefix_string, NULL, 10);
1381 - // End if the prefix was invalid
1383 - DEBUG(ctx, "The prefix is zero or not a number\n");
1387 // Parse the address
1388 r = loc_parse_address(ctx, address_string, &first_address);
1390 @@ -196,9 +182,20 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1394 - // Map the prefix to IPv6 if needed
1395 - if (IN6_IS_ADDR_V4MAPPED(&first_address))
1397 + // If a prefix was given, we will try to parse it
1398 + if (prefix_string) {
1399 + // Convert prefix to integer
1400 + prefix = strtol(prefix_string, NULL, 10);
1403 + DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
1407 + // Map the prefix to IPv6 if needed
1408 + if (IN6_IS_ADDR_V4MAPPED(&first_address))
1413 // Free temporary buffer
1414 diff --git a/src/test-network.c b/src/test-network.c
1415 index 8c7e898..b6776b4 100644
1416 --- a/src/test-network.c
1417 +++ b/src/test-network.c
1418 @@ -170,8 +170,8 @@ int main(int argc, char** argv) {
1420 // Try adding a single address
1421 err = loc_writer_add_network(writer, &network, "2001:db8::");
1422 - if (err != -EINVAL) {
1423 - fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
1425 + fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
1432 From aadac4c569e921be1d28dd3b2377ac7f3732213e Mon Sep 17 00:00:00 2001
1433 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1434 Date: Wed, 21 Oct 2020 14:47:36 +0000
1435 Subject: [PATCH 020/111] Revert "Revert "Revert "Revert "importer: Import raw
1436 sources for inetnum's again""""
1438 Content-Type: text/plain; charset=UTF-8
1439 Content-Transfer-Encoding: 8bit
1441 This reverts commit 44341478233115b26bb27fdb24da5b0a1eedb173.
1443 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1444 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1446 src/python/importer.py | 14 ++++----
1447 src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
1448 2 files changed, 70 insertions(+), 7 deletions(-)
1450 diff --git a/src/python/importer.py b/src/python/importer.py
1451 index de20f37..f19db4b 100644
1452 --- a/src/python/importer.py
1453 +++ b/src/python/importer.py
1454 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
1455 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
1457 # Asia Pacific Network Information Centre
1458 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
1459 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
1460 + "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
1461 + "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
1462 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
1463 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
1464 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
1465 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
1468 # Réseaux IP Européens
1469 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
1470 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
1471 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
1472 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
1473 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
1474 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
1475 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
1476 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
1478 EXTENDED_SOURCES = (
1479 # African Network Information Centre
1480 - "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
1481 + #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
1483 # Asia Pacific Network Information Centre
1484 - "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
1485 + #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
1487 # American Registry for Internet Numbers
1488 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
1489 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
1490 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
1492 # Réseaux IP Européens
1493 - "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
1494 + #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
1497 class Downloader(object):
1498 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1499 index 3c1e5e2..e8a4fc5 100644
1500 --- a/src/python/location-importer.in
1501 +++ b/src/python/location-importer.in
1502 @@ -405,6 +405,10 @@ class CLI(object):
1503 if line.startswith("aut-num:"):
1504 return self._parse_autnum_block(block)
1507 + if line.startswith("inet6num:") or line.startswith("inetnum:"):
1508 + return self._parse_inetnum_block(block)
1511 elif line.startswith("organisation:"):
1512 return self._parse_org_block(block)
1513 @@ -434,6 +438,65 @@ class CLI(object):
1514 autnum.get("asn"), autnum.get("org"),
1517 + def _parse_inetnum_block(self, block):
1518 + logging.debug("Parsing inetnum block:")
1521 + for line in block:
1522 + logging.debug(line)
1525 + key, val = split_line(line)
1527 + if key == "inetnum":
1528 + start_address, delim, end_address = val.partition("-")
1530 + # Strip any excess space
1531 + start_address, end_address = start_address.rstrip(), end_address.strip()
1533 + # Convert to IP address
1535 + start_address = ipaddress.ip_address(start_address)
1536 + end_address = ipaddress.ip_address(end_address)
1537 + except ValueError:
1538 + logging.warning("Could not parse line: %s" % line)
1541 + # Set prefix to default
1544 + # Count number of addresses in this subnet
1545 + num_addresses = int(end_address) - int(start_address)
1547 + prefix -= math.log(num_addresses, 2)
1549 + inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
1551 + elif key == "inet6num":
1552 + inetnum[key] = val
1554 + elif key == "country":
1555 + if val == "UNITED STATES":
1558 + inetnum[key] = val.upper()
1560 + # Skip empty objects
1564 + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
1566 + # Bail out in case we have processed a non-public IP network
1567 + if network.is_private:
1568 + logging.warning("Skipping non-globally routable network: %s" % network)
1571 + self.db.execute("INSERT INTO networks(network, country) \
1572 + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
1573 + "%s" % network, inetnum.get("country"),
1576 def _parse_org_block(self, block):
1582 From 002deb6b42ac0b3624c07e3352cebd72dc0685a2 Mon Sep 17 00:00:00 2001
1583 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1584 Date: Wed, 21 Oct 2020 14:47:37 +0000
1585 Subject: [PATCH 021/111] Revert "Revert "location-importer.in: only import
1586 relevant data from AFRINIC, APNIC and RIPE""
1588 Content-Type: text/plain; charset=UTF-8
1589 Content-Transfer-Encoding: 8bit
1591 This reverts commit 13f67f285856e8eabfeff2daf1be3aeaa36a82cc.
1593 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1594 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1596 src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
1597 1 file changed, 87 insertions(+), 2 deletions(-)
1599 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1600 index e8a4fc5..5656c41 100644
1601 --- a/src/python/location-importer.in
1602 +++ b/src/python/location-importer.in
1603 @@ -166,6 +166,7 @@ class CLI(object):
1605 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
1606 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
1607 + CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
1608 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
1611 @@ -375,6 +376,16 @@ class CLI(object):
1612 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
1614 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
1616 + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
1618 + CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
1619 + CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
1622 + # Remove all previously imported content
1623 + self.db.execute("""
1624 + TRUNCATE TABLE networks;
1627 for source in location.importer.WHOIS_SOURCES:
1628 @@ -382,6 +393,67 @@ class CLI(object):
1630 self._parse_block(block)
1632 + # Process all parsed networks from every RIR we happen to have access to,
1633 + # insert the largest network chunks into the networks table immediately...
1634 + families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
1636 + for family in (row.family for row in families):
1637 + smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
1639 + self.db.execute("INSERT INTO networks(network, country) \
1640 + SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
1642 + # ... determine any other prefixes for this network family, ...
1643 + prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
1644 + WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
1646 + # ... and insert networks with this prefix in case they provide additional
1647 + # information (i. e. subnet of a larger chunk with a different country)
1648 + for prefix in (row.prefix for row in prefixes):
1649 + self.db.execute("""
1650 + WITH candidates AS (
1657 + family(_rirdata.network) = %s
1659 + masklen(_rirdata.network) = %s
1663 + DISTINCT ON (c.network)
1666 + masklen(networks.network),
1667 + networks.country AS parent_country
1673 + c.network << networks.network
1676 + masklen(networks.network) DESC NULLS LAST
1679 + networks(network, country)
1686 + parent_country IS NULL
1688 + country <> parent_country
1689 + ON CONFLICT DO NOTHING""",
1694 INSERT INTO autnums(number, name)
1695 SELECT _autnums.number, _organizations.name FROM _autnums
1696 @@ -482,17 +554,30 @@ class CLI(object):
1697 inetnum[key] = val.upper()
1699 # Skip empty objects
1701 + if not inetnum or not "country" in inetnum:
1704 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
1706 + # Bail out in case we have processed a network covering the entire IP range, which
1707 + # is necessary to work around faulty (?) IPv6 network processing
1708 + if network.prefixlen == 0:
1709 + logging.warning("Skipping network covering the entire IP adress range: %s" % network)
1712 + # Bail out in case we have processed a network whose prefix length indicates it is
1713 + # not globally routable (we have decided not to process them at the moment, as they
1714 + # significantly enlarge our database without providing very helpful additional information)
1715 + if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
1716 + logging.info("Skipping network too small to be publicly announced: %s" % network)
1719 # Bail out in case we have processed a non-public IP network
1720 if network.is_private:
1721 logging.warning("Skipping non-globally routable network: %s" % network)
1724 - self.db.execute("INSERT INTO networks(network, country) \
1725 + self.db.execute("INSERT INTO _rirdata(network, country) \
1726 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
1727 "%s" % network, inetnum.get("country"),
1732 From 28c73fa3f4257e0a41e52af8a9643da414a6cb6f Mon Sep 17 00:00:00 2001
1733 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1734 Date: Wed, 21 Oct 2020 14:47:38 +0000
1735 Subject: [PATCH 022/111] export.py: fix exporting IP networks for crappy
1738 Content-Type: text/plain; charset=UTF-8
1739 Content-Transfer-Encoding: 8bit
1741 In contrast to the location database itself, the xt_geoip module
1742 consumes a list of IP networks for each country, and returns after the
1745 We therefore need to...
1747 (a) sort IP networks by their size, allow as precise matches as possible
1748 (b) export _any_ IP networks - including inverted subnets - to prevent
1750 (c) do the entire thing as fast as possible, consuming as less disk
1751 space as possible, which is why we can't just iterate over all /24
1754 Partially fixes: #12499
1756 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1757 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1758 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1760 src/python/export.py | 69 ++++++++++++++++++++++++++++++++++----------
1761 1 file changed, 54 insertions(+), 15 deletions(-)
1763 diff --git a/src/python/export.py b/src/python/export.py
1764 index d15c6f0..5eaf43f 100644
1765 --- a/src/python/export.py
1766 +++ b/src/python/export.py
1767 @@ -39,8 +39,8 @@ class OutputWriter(object):
1771 - def __init__(self, f, prefix=None, flatten=True):
1772 - self.f, self.prefix, self.flatten = f, prefix, flatten
1773 + def __init__(self, db, f, prefix=None, flatten=True):
1774 + self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten
1776 # The previously written network
1777 self._last_network = None
1778 @@ -49,13 +49,13 @@ class OutputWriter(object):
1779 self._write_header()
1782 - def open(cls, filename, **kwargs):
1783 + def open(cls, db, filename, **kwargs):
1785 Convenience function to open a file
1787 f = open(filename, cls.mode)
1789 - return cls(f, **kwargs)
1790 + return cls(db, f, **kwargs)
1793 return "<%s f=%s>" % (self.__class__.__name__, self.f)
1794 @@ -87,13 +87,31 @@ class OutputWriter(object):
1795 def _write_network(self, network):
1796 self.f.write("%s\n" % network)
1798 - def write(self, network):
1799 + def write(self, network, subnets):
1800 if self.flatten and self._flatten(network):
1801 log.debug("Skipping writing network %s" % network)
1804 - # Write the network to file
1805 - self._write_network(network)
1806 + # Write the network when it has no subnets
1808 + network = ipaddress.ip_network("%s" % network)
1809 + return self._write_network(network)
1811 + # Collect all matching subnets
1812 + matching_subnets = []
1814 + for subnet in sorted(subnets):
1815 + # Try to find the subnet in the database
1816 + n = self.db.lookup("%s" % subnet.network_address)
1818 + # No entry or matching country means those IP addresses belong here
1819 + if not n or n.country_code == network.country_code:
1820 + matching_subnets.append(subnet)
1822 + # Write all networks as compact as possible
1823 + for network in ipaddress.collapse_addresses(matching_subnets):
1824 + log.debug("Writing %s to database" % network)
1825 + self._write_network(network)
1829 @@ -143,10 +161,10 @@ class XTGeoIPOutputWriter(OutputWriter):
1832 def _write_network(self, network):
1833 - for address in (network.first_address, network.last_address):
1834 + for address in (network.network_address, network.broadcast_address):
1835 # Convert this into a string of bits
1836 bytes = socket.inet_pton(
1837 - network.family, address,
1838 + socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
1842 @@ -175,7 +193,7 @@ class Exporter(object):
1843 directory, prefix=country_code, suffix=self.writer.suffix, family=family,
1846 - writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
1847 + writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code)
1849 # Create writers for ASNs
1851 @@ -183,22 +201,43 @@ class Exporter(object):
1852 directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
1855 - writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
1856 + writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
1858 # Get all networks that match the family
1859 networks = self.db.search_networks(family=family)
1861 + # Materialise the generator into a list (uses quite some memory)
1862 + networks = list(networks)
1864 # Walk through all networks
1865 - for network in networks:
1866 + for i, network in enumerate(networks):
1867 + _network = ipaddress.ip_network("%s" % network)
1869 + # Search for all subnets
1872 + while i < len(networks):
1873 + subnet = networks[i+1]
1875 + if subnet.is_subnet_of(network):
1876 + _subnet = ipaddress.ip_network("%s" % subnet)
1878 + subnets.add(_subnet)
1879 + subnets.update(_network.address_exclude(_subnet))
1885 # Write matching countries
1887 - writers[network.country_code].write(network)
1888 + writers[network.country_code].write(network, subnets)
1892 # Write matching ASNs
1894 - writers[network.asn].write(network)
1895 + writers[network.asn].write(network, subnets)
1899 @@ -209,7 +248,7 @@ class Exporter(object):
1900 country = flags[flag]
1903 - writers[country].write(network)
1904 + writers[country].write(network, subnets)
1911 From bd341642fc6bbcc050e9b4ec5124585c83cab84d Mon Sep 17 00:00:00 2001
1912 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1913 Date: Wed, 21 Oct 2020 14:47:39 +0000
1914 Subject: [PATCH 023/111] location-importer.in: filter bogus IP networks for
1915 both Whois and extended sources
1917 Content-Type: text/plain; charset=UTF-8
1918 Content-Transfer-Encoding: 8bit
1920 Sanity checks for parsed networks have been put into a separate function
1921 to avoid boilerplate code for extended sources. This makes the location
1922 database less vulnerable to garbage written into RIR databases on
1923 purpose or by chance.
1927 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1928 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1930 src/python/location-importer.in | 83 ++++++++++++++++++++++++++-------
1931 1 file changed, 67 insertions(+), 16 deletions(-)
1933 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1934 index 5656c41..f24d357 100644
1935 --- a/src/python/location-importer.in
1936 +++ b/src/python/location-importer.in
1937 @@ -469,6 +469,69 @@ class CLI(object):
1939 self._parse_line(line)
1941 + def _check_parsed_network(self, network):
1943 + Assistive function to detect and subsequently sort out parsed
1944 + networks from RIR data (both Whois and so-called "extended sources"),
1945 + which are or have...
1947 + (a) not globally routable (RFC 1918 space, et al.)
1948 + (b) covering a too large chunk of the IP address space (prefix length
1949 + is < 7 for IPv4 networks, and < 10 for IPv6)
1950 + (c) "0.0.0.0" or "::" as a network address
1951 + (d) are too small for being publicly announced (we have decided not to
1952 + process them at the moment, as they significantly enlarge our
1953 + database without providing very helpful additional information)
1955 + This unfortunately is necessary due to brain-dead clutter across
1956 + various RIR databases, causing mismatches and eventually disruptions.
1958 + We will return False in case a network is not suitable for adding
1959 + it to our database, and True otherwise.
1962 + if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
1965 + if not network.is_global:
1966 + logging.warning("Skipping non-globally routable network: %s" % network)
1969 + if network.version == 4:
1970 + if network.prefixlen < 7:
1971 + logging.warning("Skipping too big IP chunk: %s" % network)
1974 + if network.prefixlen > 24:
1975 + logging.info("Skipping network too small to be publicly announced: %s" % network)
1978 + if str(network.network_address) == "0.0.0.0":
1979 + logging.warning("Skipping network based on 0.0.0.0: %s" % network)
1982 + elif network.version == 6:
1983 + if network.prefixlen < 10:
1984 + logging.warning("Skipping too big IP chunk: %s" % network)
1987 + if network.prefixlen > 48:
1988 + logging.info("Skipping network too small to be publicly announced: %s" % network)
1991 + if str(network.network_address) == "::":
1992 + logging.warning("Skipping network based on '::': %s" % network)
1996 + # This should not happen...
1997 + logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
2000 + # In case we have made it here, the network is considered to
2001 + # be suitable for libloc consumption...
2004 def _parse_block(self, block):
2005 # Get first line to find out what type of block this is
2007 @@ -559,22 +622,7 @@ class CLI(object):
2009 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
2011 - # Bail out in case we have processed a network covering the entire IP range, which
2012 - # is necessary to work around faulty (?) IPv6 network processing
2013 - if network.prefixlen == 0:
2014 - logging.warning("Skipping network covering the entire IP adress range: %s" % network)
2017 - # Bail out in case we have processed a network whose prefix length indicates it is
2018 - # not globally routable (we have decided not to process them at the moment, as they
2019 - # significantly enlarge our database without providing very helpful additional information)
2020 - if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
2021 - logging.info("Skipping network too small to be publicly announced: %s" % network)
2024 - # Bail out in case we have processed a non-public IP network
2025 - if network.is_private:
2026 - logging.warning("Skipping non-globally routable network: %s" % network)
2027 + if not self._check_parsed_network(network):
2030 self.db.execute("INSERT INTO _rirdata(network, country) \
2031 @@ -658,6 +706,9 @@ class CLI(object):
2032 log.warning("Invalid IP address: %s" % address)
2035 + if not self._check_parsed_network(network):
2038 self.db.execute("INSERT INTO networks(network, country) \
2039 VALUES(%s, %s) ON CONFLICT (network) DO \
2040 UPDATE SET country = excluded.country",
2044 From eee65490a10e0fe89b3834b8be176fc900084fa0 Mon Sep 17 00:00:00 2001
2045 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2046 Date: Wed, 21 Oct 2020 14:47:40 +0000
2047 Subject: [PATCH 024/111] importer.py: fetch LACNIC data via HTTPS
2049 Content-Type: text/plain; charset=UTF-8
2050 Content-Transfer-Encoding: 8bit
2052 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2053 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2055 src/python/importer.py | 2 +-
2056 1 file changed, 1 insertion(+), 1 deletion(-)
2058 diff --git a/src/python/importer.py b/src/python/importer.py
2059 index f19db4b..5f46bc3 100644
2060 --- a/src/python/importer.py
2061 +++ b/src/python/importer.py
2062 @@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
2063 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
2065 # Latin America and Caribbean Network Information Centre
2066 - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2067 + "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2069 # Réseaux IP Européens
2070 #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
2074 From 84187ab5436eb158529d6f5e2a38890b4af3ddb4 Mon Sep 17 00:00:00 2001
2075 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2076 Date: Wed, 21 Oct 2020 14:47:41 +0000
2077 Subject: [PATCH 025/111] location-importer.in: omit historic/orphaned RIR data
2079 Content-Type: text/plain; charset=UTF-8
2080 Content-Transfer-Encoding: 8bit
2082 Some RIRs include detailled information regarding networks not managed
2083 by or allocated to themselves, particually APNIC. We need to filter
2084 those networks (they usually have a characteristic network name) in
2085 order to prevent operational quirks or returning wrong country codes.
2088 Partially fixes: #12499
2090 Cc: Michael Tremer <michael.tremer@ipfire.org>
2091 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2092 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2094 src/python/location-importer.in | 38 +++++++++++++++++++++------------
2095 1 file changed, 24 insertions(+), 14 deletions(-)
2097 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2098 index f24d357..a869256 100644
2099 --- a/src/python/location-importer.in
2100 +++ b/src/python/location-importer.in
2101 @@ -494,38 +494,38 @@ class CLI(object):
2104 if not network.is_global:
2105 - logging.warning("Skipping non-globally routable network: %s" % network)
2106 + log.warning("Skipping non-globally routable network: %s" % network)
2109 if network.version == 4:
2110 if network.prefixlen < 7:
2111 - logging.warning("Skipping too big IP chunk: %s" % network)
2112 + log.warning("Skipping too big IP chunk: %s" % network)
2115 if network.prefixlen > 24:
2116 - logging.info("Skipping network too small to be publicly announced: %s" % network)
2117 + log.info("Skipping network too small to be publicly announced: %s" % network)
2120 if str(network.network_address) == "0.0.0.0":
2121 - logging.warning("Skipping network based on 0.0.0.0: %s" % network)
2122 + log.warning("Skipping network based on 0.0.0.0: %s" % network)
2125 elif network.version == 6:
2126 if network.prefixlen < 10:
2127 - logging.warning("Skipping too big IP chunk: %s" % network)
2128 + log.warning("Skipping too big IP chunk: %s" % network)
2131 if network.prefixlen > 48:
2132 - logging.info("Skipping network too small to be publicly announced: %s" % network)
2133 + log.info("Skipping network too small to be publicly announced: %s" % network)
2136 if str(network.network_address) == "::":
2137 - logging.warning("Skipping network based on '::': %s" % network)
2138 + log.warning("Skipping network based on '::': %s" % network)
2142 # This should not happen...
2143 - logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
2144 + log.warning("Skipping network of unknown family, this should not happen: %s" % network)
2147 # In case we have made it here, the network is considered to
2148 @@ -574,15 +574,22 @@ class CLI(object):
2151 def _parse_inetnum_block(self, block):
2152 - logging.debug("Parsing inetnum block:")
2153 + log.debug("Parsing inetnum block:")
2157 - logging.debug(line)
2161 key, val = split_line(line)
2163 + # Filter any inetnum records which are only referring to IP space
2164 + # not managed by that specific RIR...
2165 + if key == "netname":
2166 + if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
2167 + log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
2170 if key == "inetnum":
2171 start_address, delim, end_address = val.partition("-")
2173 @@ -594,7 +601,7 @@ class CLI(object):
2174 start_address = ipaddress.ip_address(start_address)
2175 end_address = ipaddress.ip_address(end_address)
2177 - logging.warning("Could not parse line: %s" % line)
2178 + log.warning("Could not parse line: %s" % line)
2181 # Set prefix to default
2182 @@ -611,15 +618,18 @@ class CLI(object):
2185 elif key == "country":
2186 - if val == "UNITED STATES":
2189 inetnum[key] = val.upper()
2191 # Skip empty objects
2192 if not inetnum or not "country" in inetnum:
2195 + # Skip objects with bogus country code 'ZZ'
2196 + if inetnum.get("country") == "ZZ":
2197 + log.warning("Skipping network with bogus country 'ZZ': %s" % \
2198 + (inetnum.get("inet6num") or inetnum.get("inetnum")))
2201 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
2203 if not self._check_parsed_network(network):
2207 From ebb087cfa30ec5ca0c96dcce66a91245c1ffc271 Mon Sep 17 00:00:00 2001
2208 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2209 Date: Wed, 21 Oct 2020 14:47:43 +0000
2210 Subject: [PATCH 026/111] location-importer.in: avoid log spam for too small
2213 Content-Type: text/plain; charset=UTF-8
2214 Content-Transfer-Encoding: 8bit
2216 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2217 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2219 src/python/location-importer.in | 4 ++--
2220 1 file changed, 2 insertions(+), 2 deletions(-)
2222 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2223 index a869256..864eab1 100644
2224 --- a/src/python/location-importer.in
2225 +++ b/src/python/location-importer.in
2226 @@ -503,7 +503,7 @@ class CLI(object):
2229 if network.prefixlen > 24:
2230 - log.info("Skipping network too small to be publicly announced: %s" % network)
2231 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2234 if str(network.network_address) == "0.0.0.0":
2235 @@ -516,7 +516,7 @@ class CLI(object):
2238 if network.prefixlen > 48:
2239 - log.info("Skipping network too small to be publicly announced: %s" % network)
2240 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2243 if str(network.network_address) == "::":
2247 From bbed1fd2330e8efa6b413dc152a1a6ef2d771aac Mon Sep 17 00:00:00 2001
2248 From: Michael Tremer <michael.tremer@ipfire.org>
2249 Date: Tue, 27 Oct 2020 17:14:30 +0000
2250 Subject: [PATCH 027/111] export: Flatten the tree before exporting it
2252 This patch removes the possibility that any IP address ranges
2253 might show up in multiple exported files.
2255 If this was content from the database:
2260 Then the IP address 10.0.1.1 would match for both countries.
2262 The algorithm will now break the larger /16 subnet apart into
2263 smaller subnets so that 10.0.1.0/24 is no longer overlapped.
2265 There was some time spent on this to make this as efficient
2268 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2270 src/python/export.py | 154 ++++++++++++++++++++++++++++++-------------
2271 1 file changed, 110 insertions(+), 44 deletions(-)
2273 diff --git a/src/python/export.py b/src/python/export.py
2274 index 5eaf43f..dd44332 100644
2275 --- a/src/python/export.py
2276 +++ b/src/python/export.py
2277 @@ -89,28 +89,55 @@ class OutputWriter(object):
2279 def write(self, network, subnets):
2280 if self.flatten and self._flatten(network):
2281 - log.debug("Skipping writing network %s" % network)
2282 + log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
2285 + # Convert network into a Python object
2286 + _network = ipaddress.ip_network("%s" % network)
2288 # Write the network when it has no subnets
2290 - network = ipaddress.ip_network("%s" % network)
2291 - return self._write_network(network)
2292 + log.debug("Writing %s to %s" % (_network, self.f))
2293 + return self._write_network(_network)
2295 + # Convert subnets into Python objects
2296 + _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
2298 + # Split the network into smaller bits so that
2299 + # we can accomodate for any gaps in it later
2301 + for _subnet in _subnets:
2303 + _network.address_exclude(_subnet)
2306 + # Clear the list of all subnets
2309 + # Check if all subnets to not overlap with anything else
2311 + subnet_to_check = to_check.pop()
2313 - # Collect all matching subnets
2314 - matching_subnets = []
2315 + for _subnet in _subnets:
2316 + # Drop this subnet if it equals one of the subnets
2317 + # or if it is subnet of one of them
2318 + if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
2321 - for subnet in sorted(subnets):
2322 - # Try to find the subnet in the database
2323 - n = self.db.lookup("%s" % subnet.network_address)
2324 + # Break it down if it overlaps
2325 + if subnet_to_check.overlaps(_subnet):
2327 + subnet_to_check.address_exclude(_subnet)
2331 - # No entry or matching country means those IP addresses belong here
2332 - if not n or n.country_code == network.country_code:
2333 - matching_subnets.append(subnet)
2334 + # Add the subnet again as it passed the check
2336 + subnets.append(subnet_to_check)
2338 # Write all networks as compact as possible
2339 - for network in ipaddress.collapse_addresses(matching_subnets):
2340 - log.debug("Writing %s to database" % network)
2341 + for network in ipaddress.collapse_addresses(subnets):
2342 + log.debug("Writing %s to %s" % (network, self.f))
2343 self._write_network(network)
2346 @@ -206,40 +233,40 @@ class Exporter(object):
2347 # Get all networks that match the family
2348 networks = self.db.search_networks(family=family)
2350 - # Materialise the generator into a list (uses quite some memory)
2351 - networks = list(networks)
2352 + # Create a stack with all networks in order where we can put items back
2353 + # again and retrieve them in the next iteration.
2354 + networks = BufferedStack(networks)
2356 # Walk through all networks
2357 - for i, network in enumerate(networks):
2358 - _network = ipaddress.ip_network("%s" % network)
2360 - # Search for all subnets
2363 - while i < len(networks):
2364 - subnet = networks[i+1]
2366 - if subnet.is_subnet_of(network):
2367 - _subnet = ipaddress.ip_network("%s" % subnet)
2369 - subnets.add(_subnet)
2370 - subnets.update(_network.address_exclude(_subnet))
2374 + for network in networks:
2375 + # Collect all networks which are a subnet of network
2377 + for subnet in networks:
2378 + # If the next subnet was not a subnet, we have to push
2379 + # it back on the stack and break this loop
2380 + if not subnet.is_subnet_of(network):
2381 + networks.push(subnet)
2384 + subnets.append(subnet)
2386 # Write matching countries
2388 - writers[network.country_code].write(network, subnets)
2391 + if network.country_code and network.country_code in writers:
2392 + # Mismatching subnets
2394 + subnet for subnet in subnets if not network.country_code == subnet.country_code
2397 + writers[network.country_code].write(network, gaps)
2399 # Write matching ASNs
2401 - writers[network.asn].write(network, subnets)
2404 + if network.asn and network.asn in writers:
2405 + # Mismatching subnets
2407 + subnet for subnet in subnets if not network.asn == subnet.asn
2410 + writers[network.asn].write(network, gaps)
2414 @@ -247,10 +274,19 @@ class Exporter(object):
2415 # Fetch the "fake" country code
2416 country = flags[flag]
2419 - writers[country].write(network, subnets)
2422 + if not country in writers:
2426 + subnet for subnet in subnets
2427 + if not subnet.has_flag(flag)
2430 + writers[country].write(network, gaps)
2432 + # Push all subnets back onto the stack
2433 + for subnet in reversed(subnets):
2434 + networks.push(subnet)
2436 # Write everything to the filesystem
2437 for writer in writers.values():
2438 @@ -262,3 +298,33 @@ class Exporter(object):
2441 return os.path.join(directory, filename)
2444 +class BufferedStack(object):
2446 + This class takes an iterator and when being iterated
2447 + over it returns objects from that iterator for as long
2450 + It additionally has a function to put an item back on
2451 + the back so that it will be returned again at the next
2454 + def __init__(self, iterator):
2455 + self.iterator = iterator
2458 + def __iter__(self):
2461 + def __next__(self):
2463 + return self.stack.pop(0)
2465 + return next(self.iterator)
2467 + def push(self, elem):
2469 + Takes an element and puts it on the stack
2471 + self.stack.insert(0, elem)
2475 From e99a72265c1ba2194b61663eda7e9f14e0083016 Mon Sep 17 00:00:00 2001
2476 From: Michael Tremer <michael.tremer@ipfire.org>
2477 Date: Wed, 28 Oct 2020 09:52:36 +0000
2478 Subject: [PATCH 028/111] location: Fix Python syntax error in verify()
2480 The database is now being opened before the requested
2481 method is called and handle_verify() wasn't updated.
2483 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2485 src/python/location.in | 8 +-------
2486 1 file changed, 1 insertion(+), 7 deletions(-)
2488 diff --git a/src/python/location.in b/src/python/location.in
2489 index 44ad726..b5e5758 100644
2490 --- a/src/python/location.in
2491 +++ b/src/python/location.in
2492 @@ -453,13 +453,7 @@ class CLI(object):
2496 - def handle_verify(self, ns):
2498 - db = location.Database(ns.database)
2499 - except FileNotFoundError as e:
2500 - log.error("%s: %s" % (ns.database, e))
2503 + def handle_verify(self, db, ns):
2504 # Verify the database
2505 with open(ns.public_key, "r") as f:
2506 if not db.verify(f):
2510 From 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 Mon Sep 17 00:00:00 2001
2511 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2512 Date: Thu, 29 Oct 2020 07:25:53 -0700
2513 Subject: [PATCH 029/111] location update: Remove double conversion of
2516 Content-Type: text/plain; charset=UTF-8
2517 Content-Transfer-Encoding: 8bit
2519 This caused that the timestamp in DNS was misinterpreted
2520 as local time and often, databases could not be downloaded.
2522 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2524 src/python/location.in | 5 +----
2525 1 file changed, 1 insertion(+), 4 deletions(-)
2527 diff --git a/src/python/location.in b/src/python/location.in
2528 index b5e5758..070640c 100644
2529 --- a/src/python/location.in
2530 +++ b/src/python/location.in
2531 @@ -421,11 +421,8 @@ class CLI(object):
2532 # Fetch the timestamp we need from DNS
2533 t = location.discover_latest_version()
2535 - # Parse timestamp into datetime format
2536 - timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
2538 # Check the version of the local database
2539 - if db and timestamp and db.created_at >= timestamp.timestamp():
2540 + if db and t and db.created_at >= t:
2541 log.info("Already on the latest version")
2547 From 60c1ac0307312614bd6980d30b44bb59b5a6ab6e Mon Sep 17 00:00:00 2001
2548 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2549 Date: Thu, 29 Oct 2020 07:36:46 -0700
2550 Subject: [PATCH 030/111] location.in: do not confuse UTC with local time zones
2552 Content-Type: text/plain; charset=UTF-8
2553 Content-Transfer-Encoding: 8bit
2555 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2557 src/python/location.in | 12 +++++-------
2558 1 file changed, 5 insertions(+), 7 deletions(-)
2560 diff --git a/src/python/location.in b/src/python/location.in
2561 index 070640c..0d09210 100644
2562 --- a/src/python/location.in
2563 +++ b/src/python/location.in
2564 @@ -398,10 +398,7 @@ class CLI(object):
2566 def handle_update(self, db, ns):
2568 - now = datetime.datetime.utcnow()
2570 - # Parse the database timestamp
2571 - t = datetime.datetime.utcfromtimestamp(db.created_at)
2574 if ns.cron == "daily":
2575 delta = datetime.timedelta(days=1)
2576 @@ -410,11 +407,12 @@ class CLI(object):
2577 elif ns.cron == "monthly":
2578 delta = datetime.timedelta(days=30)
2580 + delta = delta.total_seconds()
2582 # Check if the database has recently been updated
2583 - if t >= (now - delta):
2584 + if db.created_at >= (now - delta):
2586 - _("The database has been updated recently (%s)") % \
2587 - format_timedelta(now - t),
2588 + _("The database has been updated recently"),
2595 From e7d612e5219ef9ba612ed404e4e2c174110d3dd7 Mon Sep 17 00:00:00 2001
2596 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2597 Date: Tue, 3 Nov 2020 15:31:08 +0000
2598 Subject: [PATCH 031/111] location-importer.in: always convert organisation
2599 handles into upper cases
2601 Content-Type: text/plain; charset=UTF-8
2602 Content-Transfer-Encoding: 8bit
2606 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2607 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2609 src/python/location-importer.in | 6 ++++--
2610 1 file changed, 4 insertions(+), 2 deletions(-)
2612 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2613 index 864eab1..2dec89e 100644
2614 --- a/src/python/location-importer.in
2615 +++ b/src/python/location-importer.in
2616 @@ -560,7 +560,7 @@ class CLI(object):
2617 autnum["asn"] = m.group(2)
2621 + autnum[key] = val.upper()
2623 # Skip empty objects
2625 @@ -646,7 +646,9 @@ class CLI(object):
2627 key, val = split_line(line)
2629 - if key in ("organisation", "org-name"):
2630 + if key == "organisation":
2631 + org[key] = val.upper()
2632 + elif key == "org-name":
2635 # Skip empty objects
2639 From e96704f43acca1a8f56d9a680cce281f5e587ec5 Mon Sep 17 00:00:00 2001
2640 From: Michael Tremer <michael.tremer@ipfire.org>
2641 Date: Wed, 11 Nov 2020 21:16:45 +0000
2642 Subject: [PATCH 032/111] test: Add tests for database enumerator
2644 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2646 src/test-database.c | 53 +++++++++++++++++++++++++++++++++++++++++++++
2647 1 file changed, 53 insertions(+)
2649 diff --git a/src/test-database.c b/src/test-database.c
2650 index b4a75c4..4aef94e 100644
2651 --- a/src/test-database.c
2652 +++ b/src/test-database.c
2653 @@ -38,6 +38,14 @@ const char* DESCRIPTION =
2654 "Maecenas ut venenatis nunc.";
2655 const char* LICENSE = "CC";
2657 +const char* networks[] = {
2659 + "2001:db8:1000::/48",
2660 + "2001:db8:2000::/48",
2661 + "2001:db8:2020::/48",
2665 static int attempt_to_open(struct loc_ctx* ctx, char* path) {
2666 FILE* f = fopen(path, "r");
2668 @@ -139,6 +147,24 @@ int main(int argc, char** argv) {
2672 + struct loc_network* network = NULL;
2674 + // Add some networks
2675 + const char** n = networks;
2677 + err = loc_writer_add_network(writer, &network, *n);
2679 + fprintf(stderr, "Could not add network %s\n", *n);
2680 + exit(EXIT_FAILURE);
2684 + loc_network_set_country_code(network, "XX");
2690 FILE* f = tmpfile();
2692 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
2693 @@ -170,6 +196,33 @@ int main(int argc, char** argv) {
2698 + struct loc_database_enumerator* enumerator;
2699 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
2701 + fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
2702 + exit(EXIT_FAILURE);
2705 + // Walk through all networks
2707 + err = loc_database_enumerator_next_network(enumerator, &network);
2709 + fprintf(stderr, "Error fetching the next network: %d\n", err);
2710 + exit(EXIT_FAILURE);
2716 + char* s = loc_network_str(network);
2717 + printf("Got network: %s\n", s);
2721 + // Free the enumerator
2722 + loc_database_enumerator_unref(enumerator);
2724 // Close the database
2725 loc_database_unref(db);
2730 From ecce288da39a2c0eb60076050ca21e9619f61844 Mon Sep 17 00:00:00 2001
2731 From: Michael Tremer <michael.tremer@ipfire.org>
2732 Date: Wed, 11 Nov 2020 23:01:19 +0000
2733 Subject: [PATCH 033/111] networks: Add list to manage groups of networks
2735 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2737 src/libloc.sym | 11 ++++++
2738 src/loc/network.h | 12 ++++++
2739 src/network.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++
2740 3 files changed, 119 insertions(+)
2742 diff --git a/src/libloc.sym b/src/libloc.sym
2743 index b8296eb..6ef2d27 100644
2744 --- a/src/libloc.sym
2745 +++ b/src/libloc.sym
2746 @@ -98,6 +98,17 @@ global:
2751 + loc_network_list_clear;
2752 + loc_network_list_empty;
2753 + loc_network_list_get;
2754 + loc_network_list_new;
2755 + loc_network_list_pop;
2756 + loc_network_list_push;
2757 + loc_network_list_ref;
2758 + loc_network_list_size;
2759 + loc_network_list_unref;
2763 loc_writer_add_country;
2764 diff --git a/src/loc/network.h b/src/loc/network.h
2765 index 70c3803..fd20812 100644
2766 --- a/src/loc/network.h
2767 +++ b/src/loc/network.h
2768 @@ -56,6 +56,18 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
2770 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
2773 +struct loc_network_list;
2774 +int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
2775 +struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
2776 +struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
2777 +size_t loc_network_list_size(struct loc_network_list* list);
2778 +int loc_network_list_empty(struct loc_network_list* list);
2779 +void loc_network_list_clear(struct loc_network_list* list);
2780 +struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
2781 +int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
2782 +struct loc_network* loc_network_list_pop(struct loc_network_list* list);
2784 #ifdef LIBLOC_PRIVATE
2786 int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
2787 diff --git a/src/network.c b/src/network.c
2788 index d7b1645..c9e7979 100644
2791 @@ -746,3 +746,99 @@ LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node)
2792 LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2793 return loc_network_ref(node->network);
2798 +struct loc_network_list {
2799 + struct loc_ctx* ctx;
2802 + struct loc_network* list[1024];
2807 +LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
2808 + struct loc_network_list** list) {
2809 + struct loc_network_list* l = calloc(1, sizeof(*l));
2813 + l->ctx = loc_ref(ctx);
2816 + // Do not allow this list to grow larger than this
2817 + l->max_size = 1024;
2819 + DEBUG(l->ctx, "Network list allocated at %p\n", l);
2824 +LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
2830 +static void loc_network_list_free(struct loc_network_list* list) {
2831 + DEBUG(list->ctx, "Releasing network list at %p\n", list);
2833 + for (unsigned int i = 0; i < list->size; i++)
2834 + loc_network_unref(list->list[i]);
2836 + loc_unref(list->ctx);
2840 +LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
2844 + if (--list->refcount > 0)
2847 + loc_network_list_free(list);
2851 +LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
2852 + return list->size;
2855 +LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
2856 + return list->size == 0;
2859 +LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
2860 + for (unsigned int i = 0; i < list->size; i++)
2861 + loc_network_unref(list->list[i]);
2866 +LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
2868 + if (index >= list->size)
2871 + return loc_network_ref(list->list[index]);
2874 +LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
2875 + // Check if we have space left
2876 + if (list->size == list->max_size)
2879 + list->list[list->size++] = loc_network_ref(network);
2884 +LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
2885 + // Return nothing when empty
2886 + if (loc_network_list_empty(list))
2889 + return list->list[list->size--];
2894 From 8b2205272b7872a1458ad87811abf58609f38ad4 Mon Sep 17 00:00:00 2001
2895 From: Michael Tremer <michael.tremer@ipfire.org>
2896 Date: Thu, 12 Nov 2020 13:57:35 +0000
2897 Subject: [PATCH 034/111] networks: Add function to dump lists
2899 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2901 src/libloc.sym | 1 +
2902 src/loc/network.h | 1 +
2903 src/network.c | 14 ++++++++++++++
2904 3 files changed, 16 insertions(+)
2906 diff --git a/src/libloc.sym b/src/libloc.sym
2907 index 6ef2d27..a5641c6 100644
2908 --- a/src/libloc.sym
2909 +++ b/src/libloc.sym
2910 @@ -100,6 +100,7 @@ global:
2913 loc_network_list_clear;
2914 + loc_network_list_dump;
2915 loc_network_list_empty;
2916 loc_network_list_get;
2917 loc_network_list_new;
2918 diff --git a/src/loc/network.h b/src/loc/network.h
2919 index fd20812..44c50a4 100644
2920 --- a/src/loc/network.h
2921 +++ b/src/loc/network.h
2922 @@ -64,6 +64,7 @@ struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
2923 size_t loc_network_list_size(struct loc_network_list* list);
2924 int loc_network_list_empty(struct loc_network_list* list);
2925 void loc_network_list_clear(struct loc_network_list* list);
2926 +void loc_network_list_dump(struct loc_network_list* list);
2927 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
2928 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
2929 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
2930 diff --git a/src/network.c b/src/network.c
2931 index c9e7979..0977406 100644
2934 @@ -817,6 +817,20 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
2938 +LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
2939 + struct loc_network* network;
2942 + for (unsigned int i = 0; i < list->size; i++) {
2943 + network = list->list[i];
2945 + s = loc_network_str(network);
2947 + INFO(list->ctx, "%s\n", s);
2952 LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
2954 if (index >= list->size)
2958 From 850e75167e8e03fe8b951992c9f7bc2ccb9fb711 Mon Sep 17 00:00:00 2001
2959 From: Michael Tremer <michael.tremer@ipfire.org>
2960 Date: Thu, 12 Nov 2020 14:18:40 +0000
2961 Subject: [PATCH 035/111] network: Add functions to break network into subnets
2963 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2965 src/libloc.sym | 5 +
2966 src/loc/network.h | 6 +
2967 src/network.c | 290 ++++++++++++++++++++++++++++++++++++++++++++-
2968 src/test-network.c | 61 ++++++++++
2969 4 files changed, 361 insertions(+), 1 deletion(-)
2971 diff --git a/src/libloc.sym b/src/libloc.sym
2972 index a5641c6..fcb9ea5 100644
2973 --- a/src/libloc.sym
2974 +++ b/src/libloc.sym
2975 @@ -80,6 +80,8 @@ global:
2978 loc_network_address_family;
2980 + loc_network_exclude;
2981 loc_network_format_first_address;
2982 loc_network_format_last_address;
2983 loc_network_get_asn;
2984 @@ -96,6 +98,7 @@ global:
2985 loc_network_set_country_code;
2986 loc_network_set_flag;
2988 + loc_network_subnets;
2992 @@ -107,7 +110,9 @@ global:
2993 loc_network_list_pop;
2994 loc_network_list_push;
2995 loc_network_list_ref;
2996 + loc_network_list_reverse;
2997 loc_network_list_size;
2998 + loc_network_list_sort;
2999 loc_network_list_unref;
3002 diff --git a/src/loc/network.h b/src/loc/network.h
3003 index 44c50a4..4e51a62 100644
3004 --- a/src/loc/network.h
3005 +++ b/src/loc/network.h
3006 @@ -54,7 +54,11 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
3007 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
3008 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
3010 +int loc_network_eq(struct loc_network* self, struct loc_network* other);
3011 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
3012 +struct loc_network_list* loc_network_subnets(struct loc_network* network);
3013 +struct loc_network_list* loc_network_exclude(
3014 + struct loc_network* self, struct loc_network* other);
3017 struct loc_network_list;
3018 @@ -68,6 +72,8 @@ void loc_network_list_dump(struct loc_network_list* list);
3019 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
3020 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
3021 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
3022 +void loc_network_list_sort(struct loc_network_list* list);
3023 +void loc_network_list_reverse(struct loc_network_list* list);
3025 #ifdef LIBLOC_PRIVATE
3027 diff --git a/src/network.c b/src/network.c
3028 index 0977406..6c08070 100644
3031 @@ -97,6 +97,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
3035 +static struct in6_addr address_increment(const struct in6_addr* address) {
3036 + struct in6_addr a = *address;
3038 + for (int octet = 15; octet >= 0; octet--) {
3039 + if (a.s6_addr[octet] < 255) {
3040 + a.s6_addr[octet]++;
3043 + a.s6_addr[octet] = 0;
3050 LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
3051 struct in6_addr* address, unsigned int prefix) {
3052 // Address cannot be unspecified
3053 @@ -405,6 +420,69 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
3054 return loc_network_has_flag(network, flag);
3057 +LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
3058 +#ifdef ENABLE_DEBUG
3059 + char* n1 = loc_network_str(self);
3060 + char* n2 = loc_network_str(other);
3062 + DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
3068 + // Family must be the same
3069 + if (self->family != other->family) {
3070 + DEBUG(self->ctx, " Family mismatch\n");
3075 + // The start address must be the same
3076 + if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
3077 + DEBUG(self->ctx, " Address mismatch\n");
3082 + // The prefix length must be the same
3083 + if (self->prefix != other->prefix) {
3084 + DEBUG(self->ctx, " Prefix mismatch\n");
3088 + DEBUG(self->ctx, " Yes!\n");
3093 +static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
3094 + // Families must match
3095 + if (self->family != other->family)
3098 + int r = in6_addr_cmp(&self->first_address, &other->first_address);
3113 + if (self->prefix > other->prefix)
3120 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
3121 // If the start address of the other network is smaller than this network,
3122 // it cannot be a subnet.
3123 @@ -419,6 +497,175 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net
3127 +LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
3128 + struct loc_network_list* list;
3130 + // New prefix length
3131 + unsigned int prefix = network->prefix + 1;
3133 + // Check if the new prefix is valid
3134 + if (valid_prefix(&network->first_address, prefix))
3137 + // Create a new list with the result
3138 + int r = loc_network_list_new(network->ctx, &list);
3140 + ERROR(network->ctx, "Could not create network list: %d\n", r);
3144 + struct loc_network* subnet1 = NULL;
3145 + struct loc_network* subnet2 = NULL;
3147 + // Create the first half of the network
3148 + r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix);
3152 + // The next subnet starts after the first one
3153 + struct in6_addr first_address = address_increment(&subnet1->last_address);
3155 + // Create the second half of the network
3156 + r = loc_network_new(network->ctx, &subnet2, &first_address, prefix);
3160 + // Push the both onto the stack (in reverse order)
3161 + r = loc_network_list_push(list, subnet2);
3165 + r = loc_network_list_push(list, subnet1);
3169 + loc_network_unref(subnet1);
3170 + loc_network_unref(subnet2);
3176 + loc_network_unref(subnet1);
3179 + loc_network_unref(subnet2);
3182 + loc_network_list_unref(list);
3187 +LOC_EXPORT struct loc_network_list* loc_network_exclude(
3188 + struct loc_network* self, struct loc_network* other) {
3189 + struct loc_network_list* list;
3191 +#ifdef ENABLE_DEBUG
3192 + char* n1 = loc_network_str(self);
3193 + char* n2 = loc_network_str(other);
3195 + DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
3201 + // Family must match
3202 + if (self->family != other->family) {
3203 + DEBUG(self->ctx, "Family mismatch\n");
3208 + // Other must be a subnet of self
3209 + if (!loc_network_is_subnet_of(other, self)) {
3210 + DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
3215 + // We cannot perform this operation if both networks equal
3216 + if (loc_network_eq(self, other)) {
3217 + DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
3222 + // Create a new list with the result
3223 + int r = loc_network_list_new(self->ctx, &list);
3225 + ERROR(self->ctx, "Could not create network list: %d\n", r);
3229 + struct loc_network_list* subnets = loc_network_subnets(self);
3231 + struct loc_network* subnet1 = NULL;
3232 + struct loc_network* subnet2 = NULL;
3235 + // Fetch both subnets
3236 + subnet1 = loc_network_list_get(subnets, 0);
3237 + subnet2 = loc_network_list_get(subnets, 1);
3240 + loc_network_list_unref(subnets);
3243 + if (loc_network_eq(other, subnet1)) {
3244 + r = loc_network_list_push(list, subnet2);
3248 + } else if (loc_network_eq(other, subnet2)) {
3249 + r = loc_network_list_push(list, subnet1);
3253 + } else if (loc_network_is_subnet_of(other, subnet1)) {
3254 + r = loc_network_list_push(list, subnet2);
3258 + subnets = loc_network_subnets(subnet1);
3260 + } else if (loc_network_is_subnet_of(other, subnet2)) {
3261 + r = loc_network_list_push(list, subnet1);
3265 + subnets = loc_network_subnets(subnet2);
3268 + ERROR(self->ctx, "We should never get here\n");
3272 + loc_network_unref(subnet1);
3273 + loc_network_unref(subnet2);
3276 +#ifdef ENABLE_DEBUG
3277 + loc_network_list_dump(list);
3280 + // Return the result
3285 + loc_network_unref(subnet1);
3288 + loc_network_unref(subnet2);
3291 + loc_network_list_unref(list);
3296 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
3298 loc_country_code_copy(dbobj->country_code, network->country_code);
3299 @@ -854,5 +1101,46 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
3300 if (loc_network_list_empty(list))
3303 - return list->list[list->size--];
3304 + return list->list[--list->size];
3307 +static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
3308 + // Do nothing for invalid indices
3309 + if (i1 >= list->size || i2 >= list->size)
3312 + DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
3314 + struct loc_network* network1 = list->list[i1];
3315 + struct loc_network* network2 = list->list[i2];
3317 + list->list[i1] = network2;
3318 + list->list[i2] = network1;
3321 +LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
3322 + unsigned int i = 0;
3323 + unsigned int j = list->size - 1;
3326 + loc_network_list_swap(list, i++, j--);
3330 +LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
3331 + unsigned int n = list->size;
3337 + for (unsigned int i = 1; i < n; i++) {
3338 + if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
3339 + loc_network_list_swap(list, i-1, i);
3345 + } while (swapped);
3347 diff --git a/src/test-network.c b/src/test-network.c
3348 index b6776b4..af1b2e6 100644
3349 --- a/src/test-network.c
3350 +++ b/src/test-network.c
3351 @@ -118,6 +118,19 @@ int main(int argc, char** argv) {
3352 size_t nodes = loc_network_tree_count_nodes(tree);
3353 printf("The tree has %zu nodes\n", nodes);
3355 + // Check equals function
3356 + err = loc_network_eq(network1, network1);
3358 + fprintf(stderr, "Network is not equal with itself\n");
3359 + exit(EXIT_FAILURE);
3362 + err = loc_network_eq(network1, network2);
3364 + fprintf(stderr, "Networks equal unexpectedly\n");
3365 + exit(EXIT_FAILURE);
3368 // Check subnet function
3369 err = loc_network_is_subnet_of(network1, network2);
3371 @@ -131,6 +144,54 @@ int main(int argc, char** argv) {
3375 + // Make list of subnets
3376 + struct loc_network_list* subnets = loc_network_subnets(network1);
3378 + fprintf(stderr, "Could not find subnets of network\n");
3379 + exit(EXIT_FAILURE);
3382 + loc_network_list_dump(subnets);
3384 + while (!loc_network_list_empty(subnets)) {
3385 + struct loc_network* subnet = loc_network_list_pop(subnets);
3387 + fprintf(stderr, "Received an empty subnet\n");
3388 + exit(EXIT_FAILURE);
3391 + char* s = loc_network_str(subnet);
3392 + printf("Received subnet %s\n", s);
3395 + if (!loc_network_is_subnet_of(subnet, network1)) {
3396 + fprintf(stderr, "Not a subnet\n");
3397 + exit(EXIT_FAILURE);
3400 + loc_network_unref(subnet);
3403 + loc_network_list_unref(subnets);
3405 + struct loc_network_list* excluded = loc_network_exclude(network1, network2);
3407 + fprintf(stderr, "Could not create excluded list\n");
3408 + exit(EXIT_FAILURE);
3411 + loc_network_list_dump(excluded);
3414 + loc_network_list_reverse(excluded);
3415 + loc_network_list_dump(excluded);
3417 + // Sort them and dump them again
3418 + loc_network_list_sort(excluded);
3419 + loc_network_list_dump(excluded);
3421 + loc_network_list_unref(excluded);
3423 // Create a database
3424 struct loc_writer* writer;
3425 err = loc_writer_new(ctx, &writer, NULL, NULL);
3429 From 6159d384c4a98fe45ec52522e2950719e4982d80 Mon Sep 17 00:00:00 2001
3430 From: Michael Tremer <michael.tremer@ipfire.org>
3431 Date: Thu, 12 Nov 2020 14:24:58 +0000
3432 Subject: [PATCH 036/111] networks: Add function to check if two networks
3435 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3437 src/libloc.sym | 1 +
3438 src/loc/network.h | 1 +
3439 src/network.c | 16 ++++++++++++++++
3440 3 files changed, 18 insertions(+)
3442 diff --git a/src/libloc.sym b/src/libloc.sym
3443 index fcb9ea5..0c2835b 100644
3444 --- a/src/libloc.sym
3445 +++ b/src/libloc.sym
3446 @@ -93,6 +93,7 @@ global:
3447 loc_network_match_flag;
3449 loc_network_new_from_string;
3450 + loc_network_overlaps;
3452 loc_network_set_asn;
3453 loc_network_set_country_code;
3454 diff --git a/src/loc/network.h b/src/loc/network.h
3455 index 4e51a62..ef13756 100644
3456 --- a/src/loc/network.h
3457 +++ b/src/loc/network.h
3458 @@ -55,6 +55,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
3459 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
3461 int loc_network_eq(struct loc_network* self, struct loc_network* other);
3462 +int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
3463 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
3464 struct loc_network_list* loc_network_subnets(struct loc_network* network);
3465 struct loc_network_list* loc_network_exclude(
3466 diff --git a/src/network.c b/src/network.c
3467 index 6c08070..d826511 100644
3470 @@ -483,6 +483,22 @@ static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
3474 +LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
3475 + if (loc_network_match_address(self, &other->first_address) == 0)
3478 + if (loc_network_match_address(self, &other->last_address) == 0)
3481 + if (loc_network_match_address(other, &self->first_address) == 0)
3484 + if (loc_network_match_address(other, &self->last_address) == 0)
3490 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
3491 // If the start address of the other network is smaller than this network,
3492 // it cannot be a subnet.
3496 From e52ba21761f27e592040a2793b2a26bbeeeecc05 Mon Sep 17 00:00:00 2001
3497 From: Michael Tremer <michael.tremer@ipfire.org>
3498 Date: Thu, 12 Nov 2020 14:28:15 +0000
3499 Subject: [PATCH 037/111] networks: Add function to check if network is part of
3502 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3504 src/libloc.sym | 1 +
3505 src/loc/network.h | 1 +
3506 src/network.c | 9 +++++++++
3507 3 files changed, 11 insertions(+)
3509 diff --git a/src/libloc.sym b/src/libloc.sym
3510 index 0c2835b..f1b63a2 100644
3511 --- a/src/libloc.sym
3512 +++ b/src/libloc.sym
3513 @@ -104,6 +104,7 @@ global:
3516 loc_network_list_clear;
3517 + loc_network_list_contains;
3518 loc_network_list_dump;
3519 loc_network_list_empty;
3520 loc_network_list_get;
3521 diff --git a/src/loc/network.h b/src/loc/network.h
3522 index ef13756..7804512 100644
3523 --- a/src/loc/network.h
3524 +++ b/src/loc/network.h
3525 @@ -73,6 +73,7 @@ void loc_network_list_dump(struct loc_network_list* list);
3526 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
3527 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
3528 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
3529 +int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
3530 void loc_network_list_sort(struct loc_network_list* list);
3531 void loc_network_list_reverse(struct loc_network_list* list);
3533 diff --git a/src/network.c b/src/network.c
3534 index d826511..fcbdc59 100644
3537 @@ -1120,6 +1120,15 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
3538 return list->list[--list->size];
3541 +LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
3542 + for (unsigned int i = 0; i < list->size; i++) {
3543 + if (loc_network_eq(list->list[i], network))
3550 static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
3551 // Do nothing for invalid indices
3552 if (i1 >= list->size || i2 >= list->size)
3556 From f802f3a4decf4827ecc8bcabe269ae9f94f7f32d Mon Sep 17 00:00:00 2001
3557 From: Michael Tremer <michael.tremer@ipfire.org>
3558 Date: Thu, 12 Nov 2020 14:33:22 +0000
3559 Subject: [PATCH 038/111] networks: Add function to merge two lists
3561 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3563 src/libloc.sym | 1 +
3564 src/loc/network.h | 1 +
3565 src/network.c | 13 +++++++++++++
3566 3 files changed, 15 insertions(+)
3568 diff --git a/src/libloc.sym b/src/libloc.sym
3569 index f1b63a2..c0b6b1f 100644
3570 --- a/src/libloc.sym
3571 +++ b/src/libloc.sym
3572 @@ -108,6 +108,7 @@ global:
3573 loc_network_list_dump;
3574 loc_network_list_empty;
3575 loc_network_list_get;
3576 + loc_network_list_merge;
3577 loc_network_list_new;
3578 loc_network_list_pop;
3579 loc_network_list_push;
3580 diff --git a/src/loc/network.h b/src/loc/network.h
3581 index 7804512..e30d91c 100644
3582 --- a/src/loc/network.h
3583 +++ b/src/loc/network.h
3584 @@ -76,6 +76,7 @@ struct loc_network* loc_network_list_pop(struct loc_network_list* list);
3585 int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
3586 void loc_network_list_sort(struct loc_network_list* list);
3587 void loc_network_list_reverse(struct loc_network_list* list);
3588 +int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
3590 #ifdef LIBLOC_PRIVATE
3592 diff --git a/src/network.c b/src/network.c
3593 index fcbdc59..541286d 100644
3596 @@ -1169,3 +1169,16 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
3601 +LOC_EXPORT int loc_network_list_merge(
3602 + struct loc_network_list* self, struct loc_network_list* other) {
3605 + for (unsigned int i = 0; i < other->size; i++) {
3606 + r = loc_network_list_push(self, other->list[i]);
3616 From 6d22a179dffd08fcf2a44aafb361725ab22486d4 Mon Sep 17 00:00:00 2001
3617 From: Michael Tremer <michael.tremer@ipfire.org>
3618 Date: Thu, 12 Nov 2020 14:35:43 +0000
3619 Subject: [PATCH 039/111] network: Make lists unique
3621 Networks that are in the list won't be added again
3623 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3625 src/network.c | 4 ++++
3626 1 file changed, 4 insertions(+)
3628 diff --git a/src/network.c b/src/network.c
3629 index 541286d..44571b3 100644
3632 @@ -1103,6 +1103,10 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
3635 LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
3636 + // Do not add networks that are already on the list
3637 + if (loc_network_list_contains(list, network))
3640 // Check if we have space left
3641 if (list->size == list->max_size)
3646 From 681ff05cb7cdf230d38abf09a330a31498e265a4 Mon Sep 17 00:00:00 2001
3647 From: Michael Tremer <michael.tremer@ipfire.org>
3648 Date: Thu, 12 Nov 2020 19:21:13 +0000
3649 Subject: [PATCH 040/111] database: Pass flag to enumerator to flatten output
3651 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3653 src/database.c | 8 +++++++-
3654 src/loc/database.h | 6 +++++-
3655 src/perl/Location.xs | 2 +-
3656 src/python/database.c | 25 ++++++++++++++++++-------
3657 4 files changed, 31 insertions(+), 10 deletions(-)
3659 diff --git a/src/database.c b/src/database.c
3660 index fa1dad0..9baab33 100644
3661 --- a/src/database.c
3662 +++ b/src/database.c
3663 @@ -104,6 +104,9 @@ struct loc_database_enumerator {
3664 enum loc_network_flags flags;
3667 + // Flatten output?
3670 // Index of the AS we are looking at
3671 unsigned int as_index;
3673 @@ -933,7 +936,7 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
3676 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
3677 - struct loc_database* db, enum loc_database_enumerator_mode mode) {
3678 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
3679 struct loc_database_enumerator* e = calloc(1, sizeof(*e));
3682 @@ -944,6 +947,9 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
3686 + // Flatten output?
3687 + e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
3689 // Initialise graph search
3690 //e->network_stack[++e->network_stack_depth] = 0;
3691 e->network_stack_depth = 1;
3692 diff --git a/src/loc/database.h b/src/loc/database.h
3693 index 43173dd..14eb5ea 100644
3694 --- a/src/loc/database.h
3695 +++ b/src/loc/database.h
3696 @@ -55,9 +55,13 @@ enum loc_database_enumerator_mode {
3697 LOC_DB_ENUMERATE_COUNTRIES = 3,
3700 +enum loc_database_enumerator_flags {
3701 + LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
3704 struct loc_database_enumerator;
3705 int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
3706 - struct loc_database* db, enum loc_database_enumerator_mode mode);
3707 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
3708 struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
3709 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
3711 diff --git a/src/perl/Location.xs b/src/perl/Location.xs
3712 index dcf3f0d..b7676d2 100644
3713 --- a/src/perl/Location.xs
3714 +++ b/src/perl/Location.xs
3715 @@ -125,7 +125,7 @@ database_countries(db)
3717 // Create Database enumerator
3718 struct loc_database_enumerator* enumerator;
3719 - int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
3720 + int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
3723 croak("Could not create a database enumerator\n");
3724 diff --git a/src/python/database.c b/src/python/database.c
3725 index 1013a58..7f8c2c2 100644
3726 --- a/src/python/database.c
3727 +++ b/src/python/database.c
3728 @@ -207,10 +207,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
3729 return (PyObject*)self;
3732 -static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
3733 +static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
3734 struct loc_database_enumerator* enumerator;
3736 - int r = loc_database_enumerator_new(&enumerator, self->db, what);
3737 + int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
3739 PyErr_SetFromErrno(PyExc_SystemError);
3741 @@ -223,7 +223,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
3744 static PyObject* Database_ases(DatabaseObject* self) {
3745 - return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
3746 + return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
3749 static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
3750 @@ -234,7 +234,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
3752 struct loc_database_enumerator* enumerator;
3754 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
3755 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
3757 PyErr_SetFromErrno(PyExc_SystemError);
3759 @@ -250,7 +250,11 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
3762 static PyObject* Database_networks(DatabaseObject* self) {
3763 - return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
3764 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
3767 +static PyObject* Database_networks_flattened(DatabaseObject *self) {
3768 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
3771 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
3772 @@ -264,7 +268,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
3775 struct loc_database_enumerator* enumerator;
3776 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
3777 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
3779 PyErr_SetFromErrno(PyExc_SystemError);
3781 @@ -317,7 +321,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
3784 static PyObject* Database_countries(DatabaseObject* self) {
3785 - return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
3786 + return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
3789 static struct PyMethodDef Database_methods[] = {
3790 @@ -403,6 +407,13 @@ static struct PyGetSetDef Database_getsetters[] = {
3795 + "networks_flattened",
3796 + (getter)Database_networks_flattened,
3803 (getter)Database_get_vendor,
3807 From f5e50a47e37e9b29d0d2ee9e5a41e5a5fe5aea7f Mon Sep 17 00:00:00 2001
3808 From: Michael Tremer <michael.tremer@ipfire.org>
3809 Date: Thu, 12 Nov 2020 19:21:58 +0000
3810 Subject: [PATCH 041/111] network: Reduce debugging output
3812 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3814 src/network.c | 28 +++-------------------------
3815 1 file changed, 3 insertions(+), 25 deletions(-)
3817 diff --git a/src/network.c b/src/network.c
3818 index 44571b3..f7071a6 100644
3821 @@ -421,37 +421,17 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
3824 LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
3825 -#ifdef ENABLE_DEBUG
3826 - char* n1 = loc_network_str(self);
3827 - char* n2 = loc_network_str(other);
3829 - DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
3835 // Family must be the same
3836 - if (self->family != other->family) {
3837 - DEBUG(self->ctx, " Family mismatch\n");
3839 + if (self->family != other->family)
3843 // The start address must be the same
3844 - if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
3845 - DEBUG(self->ctx, " Address mismatch\n");
3847 + if (in6_addr_cmp(&self->first_address, &other->first_address) != 0)
3851 // The prefix length must be the same
3852 - if (self->prefix != other->prefix) {
3853 - DEBUG(self->ctx, " Prefix mismatch\n");
3854 + if (self->prefix != other->prefix)
3858 - DEBUG(self->ctx, " Yes!\n");
3862 @@ -1138,8 +1118,6 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
3863 if (i1 >= list->size || i2 >= list->size)
3866 - DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
3868 struct loc_network* network1 = list->list[i1];
3869 struct loc_network* network2 = list->list[i2];
3874 From 037c65d3a07ec6d37ff063f0645adda6b483b407 Mon Sep 17 00:00:00 2001
3875 From: Michael Tremer <michael.tremer@ipfire.org>
3876 Date: Thu, 12 Nov 2020 19:36:38 +0000
3877 Subject: [PATCH 042/111] python: Export networks exclude function
3879 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3881 src/python/network.c | 39 +++++++++++++++++++++++++++++++++++++++
3882 1 file changed, 39 insertions(+)
3884 diff --git a/src/python/network.c b/src/python/network.c
3885 index 5496d1e..11f672b 100644
3886 --- a/src/python/network.c
3887 +++ b/src/python/network.c
3889 #include "locationmodule.h"
3890 #include "network.h"
3892 +static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
3893 + PyObject* list = PyList_New(0);
3897 + while (!loc_network_list_empty(networks)) {
3898 + struct loc_network* network = loc_network_list_pop(networks);
3900 + PyObject* n = new_network(&NetworkType, network);
3901 + PyList_Append(list, n);
3903 + loc_network_unref(network);
3910 PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
3911 NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
3913 @@ -154,6 +172,21 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
3917 +static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
3918 + NetworkObject* other = NULL;
3920 + if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3923 + struct loc_network_list* list = loc_network_exclude(self->network, other->network);
3925 + // Convert to Python objects
3926 + PyObject* obj = PyList_FromNetworkList(list);
3927 + loc_network_list_unref(list);
3932 static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
3933 NetworkObject* other = NULL;
3935 @@ -191,6 +224,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
3938 static struct PyMethodDef Network_methods[] = {
3941 + (PyCFunction)Network_exclude,
3947 (PyCFunction)Network_has_flag,
3951 From 9a7732c8679e805d4d2d55ea4750c5d70ca4bd2c Mon Sep 17 00:00:00 2001
3952 From: Michael Tremer <michael.tremer@ipfire.org>
3953 Date: Thu, 12 Nov 2020 19:59:22 +0000
3954 Subject: [PATCH 043/111] network: Add more debugging output to stacks
3956 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3958 src/network.c | 16 +++++++++++++---
3959 1 file changed, 13 insertions(+), 3 deletions(-)
3961 diff --git a/src/network.c b/src/network.c
3962 index f7071a6..d41e873 100644
3965 @@ -1088,8 +1088,12 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
3968 // Check if we have space left
3969 - if (list->size == list->max_size)
3970 + if (list->size == list->max_size) {
3971 + ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
3975 + DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
3977 list->list[list->size++] = loc_network_ref(network);
3979 @@ -1098,10 +1102,16 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
3981 LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
3982 // Return nothing when empty
3983 - if (loc_network_list_empty(list))
3984 + if (loc_network_list_empty(list)) {
3985 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
3989 - return list->list[--list->size];
3990 + struct loc_network* network = list->list[--list->size];
3992 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
3997 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
4001 From 33a051e0435f6e78cc936f26f3b9ee16b7851025 Mon Sep 17 00:00:00 2001
4002 From: Michael Tremer <michael.tremer@ipfire.org>
4003 Date: Thu, 12 Nov 2020 20:00:09 +0000
4004 Subject: [PATCH 044/111] network: Add new subnet function
4006 The old one is too difficult to use in terms of order
4007 of input parameters and return value.
4009 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4011 src/libloc.sym | 1 +
4012 src/loc/network.h | 1 +
4013 src/network.c | 15 +++++++++++++++
4014 3 files changed, 17 insertions(+)
4016 diff --git a/src/libloc.sym b/src/libloc.sym
4017 index c0b6b1f..5392437 100644
4018 --- a/src/libloc.sym
4019 +++ b/src/libloc.sym
4020 @@ -87,6 +87,7 @@ global:
4021 loc_network_get_asn;
4022 loc_network_get_country_code;
4023 loc_network_has_flag;
4024 + loc_network_is_subnet;
4025 loc_network_is_subnet_of;
4026 loc_network_match_asn;
4027 loc_network_match_country_code;
4028 diff --git a/src/loc/network.h b/src/loc/network.h
4029 index e30d91c..2154cdc 100644
4030 --- a/src/loc/network.h
4031 +++ b/src/loc/network.h
4032 @@ -56,6 +56,7 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
4034 int loc_network_eq(struct loc_network* self, struct loc_network* other);
4035 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
4036 +int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
4037 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
4038 struct loc_network_list* loc_network_subnets(struct loc_network* network);
4039 struct loc_network_list* loc_network_exclude(
4040 diff --git a/src/network.c b/src/network.c
4041 index d41e873..5719111 100644
4044 @@ -479,6 +479,21 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
4048 +LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
4049 + // If the start address of the other network is smaller than this network,
4050 + // it cannot be a subnet.
4051 + if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
4054 + // If the end address of the other network is greater than this network,
4055 + // it cannot be a subnet.
4056 + if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
4062 +// XXX DEPRECATED - I find this too difficult to use
4063 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
4064 // If the start address of the other network is smaller than this network,
4065 // it cannot be a subnet.
4069 From add5bb652ba1dad1127f79cb6a0db2d363a6d5e5 Mon Sep 17 00:00:00 2001
4070 From: Michael Tremer <michael.tremer@ipfire.org>
4071 Date: Thu, 12 Nov 2020 20:01:17 +0000
4072 Subject: [PATCH 045/111] network: Add function to exclude multiple networks at
4075 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4077 src/libloc.sym | 1 +
4078 src/loc/network.h | 2 ++
4079 src/network.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++
4080 3 files changed, 88 insertions(+)
4082 diff --git a/src/libloc.sym b/src/libloc.sym
4083 index 5392437..bcd11be 100644
4084 --- a/src/libloc.sym
4085 +++ b/src/libloc.sym
4086 @@ -82,6 +82,7 @@ global:
4087 loc_network_address_family;
4089 loc_network_exclude;
4090 + loc_network_exclude_list;
4091 loc_network_format_first_address;
4092 loc_network_format_last_address;
4093 loc_network_get_asn;
4094 diff --git a/src/loc/network.h b/src/loc/network.h
4095 index 2154cdc..40712b9 100644
4096 --- a/src/loc/network.h
4097 +++ b/src/loc/network.h
4098 @@ -61,6 +61,8 @@ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other
4099 struct loc_network_list* loc_network_subnets(struct loc_network* network);
4100 struct loc_network_list* loc_network_exclude(
4101 struct loc_network* self, struct loc_network* other);
4102 +struct loc_network_list* loc_network_exclude_list(
4103 + struct loc_network* network, struct loc_network_list* list);
4106 struct loc_network_list;
4107 diff --git a/src/network.c b/src/network.c
4108 index 5719111..751e8e5 100644
4111 @@ -677,6 +677,91 @@ ERROR:
4115 +LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
4116 + struct loc_network* network, struct loc_network_list* list) {
4117 + struct loc_network_list* to_check;
4119 + // Create a new list with all networks to look at
4120 + int r = loc_network_list_new(network->ctx, &to_check);
4124 + struct loc_network* subnet = NULL;
4125 + struct loc_network_list* subnets = NULL;
4127 + for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
4128 + subnet = loc_network_list_get(list, i);
4130 + // Find all excluded networks
4131 + struct loc_network_list* excluded = loc_network_exclude(network, subnet);
4133 + // Add them all to the "to check" list
4134 + loc_network_list_merge(to_check, excluded);
4135 + loc_network_list_unref(excluded);
4139 + loc_network_unref(subnet);
4142 + r = loc_network_list_new(network->ctx, &subnets);
4144 + loc_network_list_unref(to_check);
4148 + while (!loc_network_list_empty(to_check)) {
4149 + struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
4151 + // Marks whether this subnet passed all checks
4154 + for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
4155 + subnet = loc_network_list_get(list, i);
4157 + // Drop this subnet if is is already in list
4158 + if (loc_network_eq(subnet_to_check, subnet)) {
4160 + loc_network_unref(subnet);
4164 + // Drop this subnet if is a subnet of another subnet
4165 + if (loc_network_is_subnet_of(subnet, subnet_to_check)) {
4167 + loc_network_unref(subnet);
4171 + // Break it down if it overlaps
4172 + if (loc_network_overlaps(subnet_to_check, subnet)) {
4175 + struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet);
4177 + loc_network_list_merge(to_check, excluded);
4178 + loc_network_list_unref(excluded);
4181 + loc_network_unref(subnet);
4185 + loc_network_unref(subnet);
4189 + r = loc_network_list_push(subnets, subnet_to_check);
4192 + loc_network_unref(subnet_to_check);
4195 + loc_network_list_unref(to_check);
4200 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
4202 loc_country_code_copy(dbobj->country_code, network->country_code);
4206 From d87fd7a3d277b4b03222c7d1680e51b3e45e525b Mon Sep 17 00:00:00 2001
4207 From: Michael Tremer <michael.tremer@ipfire.org>
4208 Date: Thu, 12 Nov 2020 20:02:03 +0000
4209 Subject: [PATCH 046/111] database: Add option to return networks flattened
4211 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4213 src/database.c | 174 ++++++++++++++++++++++++++++++++++++++++---------
4214 1 file changed, 143 insertions(+), 31 deletions(-)
4216 diff --git a/src/database.c b/src/database.c
4217 index 9baab33..7a3d1a7 100644
4218 --- a/src/database.c
4219 +++ b/src/database.c
4220 @@ -118,6 +118,9 @@ struct loc_database_enumerator {
4221 struct loc_node_stack network_stack[MAX_STACK_DEPTH];
4222 int network_stack_depth;
4223 unsigned int* networks_visited;
4225 + // For subnet search
4226 + struct loc_network_list* stack;
4229 static int loc_database_read_magic(struct loc_database* db) {
4230 @@ -935,6 +938,26 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
4234 +static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
4235 + DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
4237 + // Release all references
4238 + loc_database_unref(enumerator->db);
4239 + loc_unref(enumerator->ctx);
4241 + if (enumerator->string)
4242 + free(enumerator->string);
4244 + // Free network search
4245 + free(enumerator->networks_visited);
4247 + // Free subnet stack
4248 + if (enumerator->stack)
4249 + loc_network_list_unref(enumerator->stack);
4254 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
4255 struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
4256 struct loc_database_enumerator* e = calloc(1, sizeof(*e));
4257 @@ -951,10 +974,16 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
4258 e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
4260 // Initialise graph search
4261 - //e->network_stack[++e->network_stack_depth] = 0;
4262 e->network_stack_depth = 1;
4263 e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
4266 + int r = loc_network_list_new(e->ctx, &e->stack);
4268 + loc_database_enumerator_free(e);
4272 DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
4275 @@ -967,22 +996,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
4279 -static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
4280 - DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
4282 - // Release all references
4283 - loc_database_unref(enumerator->db);
4284 - loc_unref(enumerator->ctx);
4286 - if (enumerator->string)
4287 - free(enumerator->string);
4289 - // Free network search
4290 - free(enumerator->networks_visited);
4295 LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
4298 @@ -1116,17 +1129,13 @@ static int loc_database_enumerator_stack_push_node(
4302 -LOC_EXPORT int loc_database_enumerator_next_network(
4303 - struct loc_database_enumerator* enumerator, struct loc_network** network) {
4307 - // Do not do anything if not in network mode
4308 - if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
4309 +static int __loc_database_enumerator_next_network(
4310 + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
4311 + // Return top element from the stack
4312 + *network = loc_network_list_pop(enumerator->stack);
4318 DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
4319 enumerator->network_stack_depth);
4321 @@ -1155,7 +1164,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
4322 enumerator->db->network_nodes_v1 + node->offset;
4324 // Add edges to stack
4325 - r = loc_database_enumerator_stack_push_node(enumerator,
4326 + int r = loc_database_enumerator_stack_push_node(enumerator,
4327 be32toh(n->one), 1, node->depth + 1);
4330 @@ -1181,6 +1190,10 @@ LOC_EXPORT int loc_database_enumerator_next_network(
4334 + // Return all networks when the filter is disabled
4338 // Check if we are interested in this network
4340 // Skip if the family does not match
4341 @@ -1223,12 +1236,111 @@ LOC_EXPORT int loc_database_enumerator_next_network(
4344 // Reached the end of the search
4348 - // Mark all nodes as non-visited
4349 - for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
4350 - enumerator->networks_visited[i] = 0;
4351 +static int __loc_database_enumerator_next_network_flattened(
4352 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
4353 + // Fetch the next network
4354 + int r = __loc_database_enumerator_next_network(enumerator, network, 1);
4359 + // End if we could not read another network
4363 + struct loc_network* subnet = NULL;
4364 + struct loc_network_list* subnets;
4366 + // Create a list with all subnets
4367 + r = loc_network_list_new(enumerator->ctx, &subnets);
4371 + // Search all subnets from the database
4373 + // Fetch the next network in line
4374 + r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
4378 + // End if we did not receive another subnet
4382 + // Collect all subnets in a list
4383 + if (loc_network_is_subnet(*network, subnet)) {
4384 + r = loc_network_list_push(subnets, subnet);
4388 + loc_network_unref(subnet);
4392 + // If this is not a subnet, we push it back onto the stack and break
4393 + r = loc_network_list_push(enumerator->stack, subnet);
4397 + loc_network_unref(subnet);
4401 + DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
4403 + // We can abort here if the network has no subnets
4404 + if (loc_network_list_empty(subnets)) {
4405 + loc_network_list_unref(subnets);
4410 + // If the network has any subnets, we will break it into smaller parts
4411 + // without the subnets.
4412 + struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
4413 + if (!excluded || loc_network_list_empty(excluded)) {
4418 + // Sort the result
4419 + loc_network_list_sort(excluded);
4421 + // Reverse the list
4422 + loc_network_list_reverse(excluded);
4424 + // Replace network with the first one
4425 + loc_network_unref(*network);
4427 + *network = loc_network_list_pop(excluded);
4429 + // Push the rest onto the stack
4430 + loc_network_list_merge(enumerator->stack, excluded);
4432 + loc_network_list_unref(excluded);
4436 + loc_network_unref(subnet);
4438 + loc_network_list_unref(subnets);
4443 +LOC_EXPORT int loc_database_enumerator_next_network(
4444 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
4445 + // Do not do anything if not in network mode
4446 + if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
4449 + // Flatten output?
4450 + if (enumerator->flatten)
4451 + return __loc_database_enumerator_next_network_flattened(enumerator, network);
4453 + return __loc_database_enumerator_next_network(enumerator, network, 1);
4456 LOC_EXPORT int loc_database_enumerator_next_country(
4460 From d3ae93c27dcd7f6984fdc29cc141621e277f2e2a Mon Sep 17 00:00:00 2001
4461 From: Michael Tremer <michael.tremer@ipfire.org>
4462 Date: Thu, 12 Nov 2020 20:09:20 +0000
4463 Subject: [PATCH 047/111] test: Update API
4465 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4467 src/test-as.c | 2 +-
4468 src/test-database.c | 2 +-
4469 2 files changed, 2 insertions(+), 2 deletions(-)
4471 diff --git a/src/test-as.c b/src/test-as.c
4472 index 839a04c..2d61675 100644
4475 @@ -95,7 +95,7 @@ int main(int argc, char** argv) {
4478 struct loc_database_enumerator* enumerator;
4479 - err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
4480 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
4482 fprintf(stderr, "Could not create a database enumerator\n");
4484 diff --git a/src/test-database.c b/src/test-database.c
4485 index 4aef94e..da4b11c 100644
4486 --- a/src/test-database.c
4487 +++ b/src/test-database.c
4488 @@ -198,7 +198,7 @@ int main(int argc, char** argv) {
4491 struct loc_database_enumerator* enumerator;
4492 - err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
4493 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
4495 fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
4500 From 594ca328c6e124d0f1eb543e9c8d9bbfe8a7b628 Mon Sep 17 00:00:00 2001
4501 From: Michael Tremer <michael.tremer@ipfire.org>
4502 Date: Thu, 12 Nov 2020 20:09:37 +0000
4503 Subject: [PATCH 048/111] networks: Copy all attributes when splitting networks
4505 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4507 src/network.c | 14 ++++++++++++++
4508 1 file changed, 14 insertions(+)
4510 diff --git a/src/network.c b/src/network.c
4511 index 751e8e5..d67f116 100644
4514 @@ -550,6 +550,20 @@ LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* netw
4518 + // Copy country code
4519 + const char* country_code = loc_network_get_country_code(network);
4520 + if (country_code) {
4521 + loc_network_set_country_code(subnet1, country_code);
4522 + loc_network_set_country_code(subnet2, country_code);
4526 + uint32_t asn = loc_network_get_asn(network);
4528 + loc_network_set_asn(subnet1, asn);
4529 + loc_network_set_asn(subnet2, asn);
4532 loc_network_unref(subnet1);
4533 loc_network_unref(subnet2);
4538 From 69248038292e9ea1a4ee8912cdfc8700456753ad Mon Sep 17 00:00:00 2001
4539 From: Michael Tremer <michael.tremer@ipfire.org>
4540 Date: Fri, 13 Nov 2020 11:23:33 +0000
4541 Subject: [PATCH 049/111] database: Move network filtering into a separate
4544 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4546 src/database.c | 56 +++++++++++++++++++++++---------------------------
4547 1 file changed, 26 insertions(+), 30 deletions(-)
4549 diff --git a/src/database.c b/src/database.c
4550 index 7a3d1a7..72bc8eb 100644
4551 --- a/src/database.c
4552 +++ b/src/database.c
4553 @@ -1129,6 +1129,31 @@ static int loc_database_enumerator_stack_push_node(
4557 +static int loc_database_enumerator_filter_network(
4558 + struct loc_database_enumerator* enumerator, struct loc_network* network) {
4559 + // Skip if the family does not match
4560 + if (enumerator->family && loc_network_address_family(network) != enumerator->family)
4563 + // Skip if the country code does not match
4564 + if (*enumerator->country_code &&
4565 + !loc_network_match_country_code(network, enumerator->country_code))
4568 + // Skip if the ASN does not match
4569 + if (enumerator->asn &&
4570 + !loc_network_match_asn(network, enumerator->asn))
4573 + // Skip if flags do not match
4574 + if (enumerator->flags &&
4575 + !loc_network_match_flag(network, enumerator->flags))
4582 static int __loc_database_enumerator_next_network(
4583 struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
4584 // Return top element from the stack
4585 @@ -1195,36 +1220,7 @@ static int __loc_database_enumerator_next_network(
4588 // Check if we are interested in this network
4590 - // Skip if the family does not match
4591 - if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
4592 - loc_network_unref(*network);
4598 - // Skip if the country code does not match
4599 - if (*enumerator->country_code &&
4600 - !loc_network_match_country_code(*network, enumerator->country_code)) {
4601 - loc_network_unref(*network);
4607 - // Skip if the ASN does not match
4608 - if (enumerator->asn &&
4609 - !loc_network_match_asn(*network, enumerator->asn)) {
4610 - loc_network_unref(*network);
4616 - // Skip if flags do not match
4617 - if (enumerator->flags &&
4618 - !loc_network_match_flag(*network, enumerator->flags)) {
4619 + if (loc_database_enumerator_filter_network(enumerator, *network)) {
4620 loc_network_unref(*network);
4626 From 2113e71bf7b997c82670c5c22cf91aa6442fe6f3 Mon Sep 17 00:00:00 2001
4627 From: Michael Tremer <michael.tremer@ipfire.org>
4628 Date: Fri, 13 Nov 2020 11:29:02 +0000
4629 Subject: [PATCH 050/111] database: Filter results coming from stack
4631 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4633 src/database.c | 18 ++++++++++++++++--
4634 1 file changed, 16 insertions(+), 2 deletions(-)
4636 diff --git a/src/database.c b/src/database.c
4637 index 72bc8eb..0f3cdc2 100644
4638 --- a/src/database.c
4639 +++ b/src/database.c
4640 @@ -1157,9 +1157,23 @@ static int loc_database_enumerator_filter_network(
4641 static int __loc_database_enumerator_next_network(
4642 struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
4643 // Return top element from the stack
4644 - *network = loc_network_list_pop(enumerator->stack);
4647 + *network = loc_network_list_pop(enumerator->stack);
4653 + // Throw away any networks by filter
4654 + if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
4655 + loc_network_unref(*network);
4664 DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
4665 enumerator->network_stack_depth);
4669 From d33753688138c9938743dafbbdddf220dd2afd14 Mon Sep 17 00:00:00 2001
4670 From: Michael Tremer <michael.tremer@ipfire.org>
4671 Date: Fri, 13 Nov 2020 11:29:15 +0000
4672 Subject: [PATCH 051/111] network: Sort result of excluded lists
4674 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4676 src/database.c | 3 ---
4677 src/network.c | 3 +++
4678 2 files changed, 3 insertions(+), 3 deletions(-)
4680 diff --git a/src/database.c b/src/database.c
4681 index 0f3cdc2..6849d97 100644
4682 --- a/src/database.c
4683 +++ b/src/database.c
4684 @@ -1315,9 +1315,6 @@ static int __loc_database_enumerator_next_network_flattened(
4688 - // Sort the result
4689 - loc_network_list_sort(excluded);
4692 loc_network_list_reverse(excluded);
4694 diff --git a/src/network.c b/src/network.c
4695 index d67f116..9d02bf8 100644
4698 @@ -773,6 +773,9 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
4700 loc_network_list_unref(to_check);
4702 + // Sort the result
4703 + loc_network_list_sort(subnets);
4711 From 8d777f12f7ffa4df1b28d197563888296803b727 Mon Sep 17 00:00:00 2001
4712 From: Michael Tremer <michael.tremer@ipfire.org>
4713 Date: Fri, 13 Nov 2020 11:38:15 +0000
4714 Subject: [PATCH 052/111] network: Add function to pop first element from stack
4716 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4718 src/database.c | 6 ++----
4719 src/libloc.sym | 1 +
4720 src/loc/network.h | 1 +
4721 src/network.c | 19 +++++++++++++++++++
4722 4 files changed, 23 insertions(+), 4 deletions(-)
4724 diff --git a/src/database.c b/src/database.c
4725 index 6849d97..b9d870f 100644
4726 --- a/src/database.c
4727 +++ b/src/database.c
4728 @@ -1315,15 +1315,13 @@ static int __loc_database_enumerator_next_network_flattened(
4732 - // Reverse the list
4733 - loc_network_list_reverse(excluded);
4735 // Replace network with the first one
4736 loc_network_unref(*network);
4738 - *network = loc_network_list_pop(excluded);
4739 + *network = loc_network_list_pop_first(excluded);
4741 // Push the rest onto the stack
4742 + loc_network_list_reverse(excluded);
4743 loc_network_list_merge(enumerator->stack, excluded);
4745 loc_network_list_unref(excluded);
4746 diff --git a/src/libloc.sym b/src/libloc.sym
4747 index bcd11be..6139db6 100644
4748 --- a/src/libloc.sym
4749 +++ b/src/libloc.sym
4750 @@ -113,6 +113,7 @@ global:
4751 loc_network_list_merge;
4752 loc_network_list_new;
4753 loc_network_list_pop;
4754 + loc_network_list_pop_first;
4755 loc_network_list_push;
4756 loc_network_list_ref;
4757 loc_network_list_reverse;
4758 diff --git a/src/loc/network.h b/src/loc/network.h
4759 index 40712b9..203e61c 100644
4760 --- a/src/loc/network.h
4761 +++ b/src/loc/network.h
4762 @@ -76,6 +76,7 @@ void loc_network_list_dump(struct loc_network_list* list);
4763 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
4764 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
4765 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
4766 +struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
4767 int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
4768 void loc_network_list_sort(struct loc_network_list* list);
4769 void loc_network_list_reverse(struct loc_network_list* list);
4770 diff --git a/src/network.c b/src/network.c
4771 index 9d02bf8..e7dc97e 100644
4774 @@ -1231,6 +1231,25 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
4778 +LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
4779 + // Return nothing when empty
4780 + if (loc_network_list_empty(list)) {
4781 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
4785 + struct loc_network* network = list->list[0];
4787 + // Move all elements to the top of the stack
4788 + for (unsigned int i = 0; i < --list->size; i++) {
4789 + list->list[i] = list->list[i+1];
4792 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
4797 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
4798 for (unsigned int i = 0; i < list->size; i++) {
4799 if (loc_network_eq(list->list[i], network))
4803 From 7933f5bfb4dd7603cb646e192840762bf6394292 Mon Sep 17 00:00:00 2001
4804 From: Michael Tremer <michael.tremer@ipfire.org>
4805 Date: Fri, 13 Nov 2020 11:43:53 +0000
4806 Subject: [PATCH 053/111] network: Unexport all tree functions
4808 These should not be exported
4810 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4812 src/network.c | 28 ++++++++++++++--------------
4813 src/test-network.c | 9 +++++++++
4814 2 files changed, 23 insertions(+), 14 deletions(-)
4816 diff --git a/src/network.c b/src/network.c
4817 index e7dc97e..d015579 100644
4820 @@ -847,7 +847,7 @@ struct loc_network_tree_node {
4821 struct loc_network* network;
4824 -LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
4825 +int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
4826 struct loc_network_tree* t = calloc(1, sizeof(*t));
4829 @@ -867,7 +867,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
4833 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
4834 +struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
4835 return loc_network_tree_node_ref(tree->root);
4838 @@ -939,7 +939,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
4842 -LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
4843 +int loc_network_tree_walk(struct loc_network_tree* tree,
4844 int(*filter_callback)(struct loc_network* network, void* data),
4845 int(*callback)(struct loc_network* network, void* data), void* data) {
4846 return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
4847 @@ -954,7 +954,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
4851 -LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
4852 +struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
4853 if (--tree->refcount > 0)
4856 @@ -975,13 +975,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
4860 -LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
4861 +int loc_network_tree_dump(struct loc_network_tree* tree) {
4862 DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
4864 return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
4867 -LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
4868 +int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
4869 DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
4871 struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
4872 @@ -1012,7 +1012,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
4876 -LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
4877 +size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
4880 int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
4881 @@ -1034,11 +1034,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
4885 -LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
4886 +size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
4887 return __loc_network_tree_count_nodes(tree->root);
4890 -LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
4891 +int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
4892 struct loc_network_tree_node* n = calloc(1, sizeof(*n));
4895 @@ -1053,7 +1053,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
4899 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
4900 +struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
4904 @@ -1076,7 +1076,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
4908 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
4909 +struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
4913 @@ -1087,7 +1087,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
4917 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
4918 +struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
4922 @@ -1099,11 +1099,11 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
4923 return loc_network_tree_node_ref(node);
4926 -LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
4927 +int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
4928 return (!!node->network);
4931 -LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
4932 +struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
4933 return loc_network_ref(node->network);
4936 diff --git a/src/test-network.c b/src/test-network.c
4937 index af1b2e6..7c90224 100644
4938 --- a/src/test-network.c
4939 +++ b/src/test-network.c
4940 @@ -37,12 +37,14 @@ int main(int argc, char** argv) {
4941 // Enable debug logging
4942 loc_set_log_priority(ctx, LOG_DEBUG);
4945 struct loc_network_tree* tree;
4946 err = loc_network_tree_new(ctx, &tree);
4948 fprintf(stderr, "Could not create the network tree\n");
4954 struct loc_network* network1;
4955 @@ -58,12 +60,14 @@ int main(int argc, char** argv) {
4960 // Adding network to the tree
4961 err = loc_network_tree_add_network(tree, network1);
4963 fprintf(stderr, "Could not add network to the tree\n");
4968 // Check if the first and last addresses are correct
4969 char* string = loc_network_format_first_address(network1);
4970 @@ -101,6 +105,7 @@ int main(int argc, char** argv) {
4975 // Adding network to the tree
4976 err = loc_network_tree_add_network(tree, network2);
4978 @@ -117,6 +122,7 @@ int main(int argc, char** argv) {
4980 size_t nodes = loc_network_tree_count_nodes(tree);
4981 printf("The tree has %zu nodes\n", nodes);
4984 // Check equals function
4985 err = loc_network_eq(network1, network1);
4986 @@ -260,7 +266,10 @@ int main(int argc, char** argv) {
4987 loc_network_unref(network2);
4988 loc_network_unref(network3);
4989 loc_network_unref(network4);
4992 loc_network_tree_unref(tree);
4995 // And open it again from disk
4996 struct loc_database* db;
5000 From c242f7325bd6fc4ba26047ac24196d1c161c6e01 Mon Sep 17 00:00:00 2001
5001 From: Michael Tremer <michael.tremer@ipfire.org>
5002 Date: Fri, 13 Nov 2020 12:09:03 +0000
5003 Subject: [PATCH 054/111] python: Move tree flattening into C
5005 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
5007 src/python/database.c | 8 ++-
5008 src/python/export.py | 138 +++++-------------------------------------
5009 2 files changed, 21 insertions(+), 125 deletions(-)
5011 diff --git a/src/python/database.c b/src/python/database.c
5012 index 7f8c2c2..d169547 100644
5013 --- a/src/python/database.c
5014 +++ b/src/python/database.c
5015 @@ -258,17 +258,19 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
5018 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
5019 - char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
5020 + char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
5021 const char* country_code = NULL;
5022 unsigned int asn = 0;
5027 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
5028 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
5031 struct loc_database_enumerator* enumerator;
5032 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
5033 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
5034 + (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
5036 PyErr_SetFromErrno(PyExc_SystemError);
5038 diff --git a/src/python/export.py b/src/python/export.py
5039 index dd44332..be4a68e 100644
5040 --- a/src/python/export.py
5041 +++ b/src/python/export.py
5042 @@ -87,58 +87,12 @@ class OutputWriter(object):
5043 def _write_network(self, network):
5044 self.f.write("%s\n" % network)
5046 - def write(self, network, subnets):
5047 + def write(self, network):
5048 if self.flatten and self._flatten(network):
5049 log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
5052 - # Convert network into a Python object
5053 - _network = ipaddress.ip_network("%s" % network)
5055 - # Write the network when it has no subnets
5057 - log.debug("Writing %s to %s" % (_network, self.f))
5058 - return self._write_network(_network)
5060 - # Convert subnets into Python objects
5061 - _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
5063 - # Split the network into smaller bits so that
5064 - # we can accomodate for any gaps in it later
5066 - for _subnet in _subnets:
5068 - _network.address_exclude(_subnet)
5071 - # Clear the list of all subnets
5074 - # Check if all subnets to not overlap with anything else
5076 - subnet_to_check = to_check.pop()
5078 - for _subnet in _subnets:
5079 - # Drop this subnet if it equals one of the subnets
5080 - # or if it is subnet of one of them
5081 - if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
5084 - # Break it down if it overlaps
5085 - if subnet_to_check.overlaps(_subnet):
5087 - subnet_to_check.address_exclude(_subnet)
5091 - # Add the subnet again as it passed the check
5093 - subnets.append(subnet_to_check)
5095 - # Write all networks as compact as possible
5096 - for network in ipaddress.collapse_addresses(subnets):
5097 - log.debug("Writing %s to %s" % (network, self.f))
5098 - self._write_network(network)
5099 + return self._write_network(network)
5103 @@ -188,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter):
5106 def _write_network(self, network):
5107 - for address in (network.network_address, network.broadcast_address):
5108 + for address in (network.first_address, network.last_address):
5109 # Convert this into a string of bits
5110 bytes = socket.inet_pton(
5111 socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
5112 @@ -231,42 +185,21 @@ class Exporter(object):
5113 writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
5115 # Get all networks that match the family
5116 - networks = self.db.search_networks(family=family)
5118 - # Create a stack with all networks in order where we can put items back
5119 - # again and retrieve them in the next iteration.
5120 - networks = BufferedStack(networks)
5121 + networks = self.db.search_networks(family=family, flatten=True)
5123 # Walk through all networks
5124 for network in networks:
5125 - # Collect all networks which are a subnet of network
5127 - for subnet in networks:
5128 - # If the next subnet was not a subnet, we have to push
5129 - # it back on the stack and break this loop
5130 - if not subnet.is_subnet_of(network):
5131 - networks.push(subnet)
5134 - subnets.append(subnet)
5136 # Write matching countries
5137 - if network.country_code and network.country_code in writers:
5138 - # Mismatching subnets
5140 - subnet for subnet in subnets if not network.country_code == subnet.country_code
5143 - writers[network.country_code].write(network, gaps)
5145 + writers[network.country_code].write(network)
5149 # Write matching ASNs
5150 - if network.asn and network.asn in writers:
5151 - # Mismatching subnets
5153 - subnet for subnet in subnets if not network.asn == subnet.asn
5156 - writers[network.asn].write(network, gaps)
5158 + writers[network.asn].write(network)
5164 @@ -274,19 +207,10 @@ class Exporter(object):
5165 # Fetch the "fake" country code
5166 country = flags[flag]
5168 - if not country in writers:
5172 - subnet for subnet in subnets
5173 - if not subnet.has_flag(flag)
5176 - writers[country].write(network, gaps)
5178 - # Push all subnets back onto the stack
5179 - for subnet in reversed(subnets):
5180 - networks.push(subnet)
5182 + writers[country].write(network)
5186 # Write everything to the filesystem
5187 for writer in writers.values():
5188 @@ -298,33 +222,3 @@ class Exporter(object):
5191 return os.path.join(directory, filename)
5194 -class BufferedStack(object):
5196 - This class takes an iterator and when being iterated
5197 - over it returns objects from that iterator for as long
5200 - It additionally has a function to put an item back on
5201 - the back so that it will be returned again at the next
5204 - def __init__(self, iterator):
5205 - self.iterator = iterator
5208 - def __iter__(self):
5211 - def __next__(self):
5213 - return self.stack.pop(0)
5215 - return next(self.iterator)
5217 - def push(self, elem):
5219 - Takes an element and puts it on the stack
5221 - self.stack.insert(0, elem)
5225 From e0b9ff5f38beb0d560b16db881647e5a75127df1 Mon Sep 17 00:00:00 2001
5226 From: Michael Tremer <michael.tremer@ipfire.org>
5227 Date: Sun, 15 Nov 2020 15:02:28 +0000
5228 Subject: [PATCH 055/111] Move network lists into an own file
5230 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
5233 src/libloc.sym | 1 +
5234 src/loc/network-list.h | 37 +++++++
5235 src/loc/network.h | 20 +---
5236 src/network-list.c | 224 +++++++++++++++++++++++++++++++++++++++++
5237 src/network.c | 207 +------------------------------------
5238 src/python/network.c | 1 +
5239 7 files changed, 269 insertions(+), 223 deletions(-)
5240 create mode 100644 src/loc/network-list.h
5241 create mode 100644 src/network-list.c
5243 diff --git a/Makefile.am b/Makefile.am
5244 index a0431a6..f0d8c4c 100644
5247 @@ -96,6 +96,7 @@ pkginclude_HEADERS = \
5248 src/loc/database.h \
5251 + src/loc/network-list.h \
5253 src/loc/stringpool.h \
5255 @@ -110,6 +111,7 @@ src_libloc_la_SOURCES = \
5259 + src/network-list.c \
5263 diff --git a/src/libloc.sym b/src/libloc.sym
5264 index 6139db6..453a1be 100644
5265 --- a/src/libloc.sym
5266 +++ b/src/libloc.sym
5267 @@ -87,6 +87,7 @@ global:
5268 loc_network_format_last_address;
5269 loc_network_get_asn;
5270 loc_network_get_country_code;
5272 loc_network_has_flag;
5273 loc_network_is_subnet;
5274 loc_network_is_subnet_of;
5275 diff --git a/src/loc/network-list.h b/src/loc/network-list.h
5276 new file mode 100644
5277 index 0000000..af3b28d
5279 +++ b/src/loc/network-list.h
5282 + libloc - A library to determine the location of someone on the Internet
5284 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
5286 + This library is free software; you can redistribute it and/or
5287 + modify it under the terms of the GNU Lesser General Public
5288 + License as published by the Free Software Foundation; either
5289 + version 2.1 of the License, or (at your option) any later version.
5291 + This library is distributed in the hope that it will be useful,
5292 + but WITHOUT ANY WARRANTY; without even the implied warranty of
5293 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5294 + Lesser General Public License for more details.
5297 +#ifndef LIBLOC_NETWORK_LIST_H
5298 +#define LIBLOC_NETWORK_LIST_H
5300 +struct loc_network_list;
5301 +int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
5302 +struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
5303 +struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
5304 +size_t loc_network_list_size(struct loc_network_list* list);
5305 +int loc_network_list_empty(struct loc_network_list* list);
5306 +void loc_network_list_clear(struct loc_network_list* list);
5307 +void loc_network_list_dump(struct loc_network_list* list);
5308 +struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
5309 +int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
5310 +struct loc_network* loc_network_list_pop(struct loc_network_list* list);
5311 +struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
5312 +int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
5313 +void loc_network_list_sort(struct loc_network_list* list);
5314 +void loc_network_list_reverse(struct loc_network_list* list);
5315 +int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
5318 diff --git a/src/loc/network.h b/src/loc/network.h
5319 index 203e61c..d86b685 100644
5320 --- a/src/loc/network.h
5321 +++ b/src/loc/network.h
5324 #include <loc/libloc.h>
5325 #include <loc/format.h>
5326 +#include <loc/network-list.h>
5328 enum loc_network_flags {
5329 LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
5330 @@ -55,6 +56,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
5331 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
5333 int loc_network_eq(struct loc_network* self, struct loc_network* other);
5334 +int loc_network_gt(struct loc_network* self, struct loc_network* other);
5335 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
5336 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
5337 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
5338 @@ -64,24 +66,6 @@ struct loc_network_list* loc_network_exclude(
5339 struct loc_network_list* loc_network_exclude_list(
5340 struct loc_network* network, struct loc_network_list* list);
5343 -struct loc_network_list;
5344 -int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
5345 -struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
5346 -struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
5347 -size_t loc_network_list_size(struct loc_network_list* list);
5348 -int loc_network_list_empty(struct loc_network_list* list);
5349 -void loc_network_list_clear(struct loc_network_list* list);
5350 -void loc_network_list_dump(struct loc_network_list* list);
5351 -struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
5352 -int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
5353 -struct loc_network* loc_network_list_pop(struct loc_network_list* list);
5354 -struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
5355 -int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
5356 -void loc_network_list_sort(struct loc_network_list* list);
5357 -void loc_network_list_reverse(struct loc_network_list* list);
5358 -int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
5360 #ifdef LIBLOC_PRIVATE
5362 int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
5363 diff --git a/src/network-list.c b/src/network-list.c
5364 new file mode 100644
5365 index 0000000..1f6e80e
5367 +++ b/src/network-list.c
5370 + libloc - A library to determine the location of someone on the Internet
5372 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
5374 + This library is free software; you can redistribute it and/or
5375 + modify it under the terms of the GNU Lesser General Public
5376 + License as published by the Free Software Foundation; either
5377 + version 2.1 of the License, or (at your option) any later version.
5379 + This library is distributed in the hope that it will be useful,
5380 + but WITHOUT ANY WARRANTY; without even the implied warranty of
5381 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5382 + Lesser General Public License for more details.
5386 +#include <stdlib.h>
5388 +#include <loc/libloc.h>
5389 +#include <loc/network.h>
5390 +#include <loc/private.h>
5392 +struct loc_network_list {
5393 + struct loc_ctx* ctx;
5396 + struct loc_network* list[1024];
5401 +LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
5402 + struct loc_network_list** list) {
5403 + struct loc_network_list* l = calloc(1, sizeof(*l));
5407 + l->ctx = loc_ref(ctx);
5410 + // Do not allow this list to grow larger than this
5411 + l->max_size = 1024;
5413 + DEBUG(l->ctx, "Network list allocated at %p\n", l);
5418 +LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
5424 +static void loc_network_list_free(struct loc_network_list* list) {
5425 + DEBUG(list->ctx, "Releasing network list at %p\n", list);
5427 + for (unsigned int i = 0; i < list->size; i++)
5428 + loc_network_unref(list->list[i]);
5430 + loc_unref(list->ctx);
5434 +LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
5438 + if (--list->refcount > 0)
5441 + loc_network_list_free(list);
5445 +LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
5446 + return list->size;
5449 +LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
5450 + return list->size == 0;
5453 +LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
5454 + for (unsigned int i = 0; i < list->size; i++)
5455 + loc_network_unref(list->list[i]);
5460 +LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
5461 + struct loc_network* network;
5464 + for (unsigned int i = 0; i < list->size; i++) {
5465 + network = list->list[i];
5467 + s = loc_network_str(network);
5469 + INFO(list->ctx, "%s\n", s);
5474 +LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
5476 + if (index >= list->size)
5479 + return loc_network_ref(list->list[index]);
5482 +LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
5483 + // Do not add networks that are already on the list
5484 + if (loc_network_list_contains(list, network))
5487 + // Check if we have space left
5488 + if (list->size == list->max_size) {
5489 + ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
5493 + DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
5495 + list->list[list->size++] = loc_network_ref(network);
5500 +LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
5501 + // Return nothing when empty
5502 + if (loc_network_list_empty(list)) {
5503 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5507 + struct loc_network* network = list->list[--list->size];
5509 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5514 +LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
5515 + // Return nothing when empty
5516 + if (loc_network_list_empty(list)) {
5517 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5521 + struct loc_network* network = list->list[0];
5523 + // Move all elements to the top of the stack
5524 + for (unsigned int i = 0; i < --list->size; i++) {
5525 + list->list[i] = list->list[i+1];
5528 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5533 +LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
5534 + for (unsigned int i = 0; i < list->size; i++) {
5535 + if (loc_network_eq(list->list[i], network))
5542 +static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
5543 + // Do nothing for invalid indices
5544 + if (i1 >= list->size || i2 >= list->size)
5547 + struct loc_network* network1 = list->list[i1];
5548 + struct loc_network* network2 = list->list[i2];
5550 + list->list[i1] = network2;
5551 + list->list[i2] = network1;
5554 +LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
5555 + unsigned int i = 0;
5556 + unsigned int j = list->size - 1;
5559 + loc_network_list_swap(list, i++, j--);
5563 +LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
5564 + unsigned int n = list->size;
5570 + for (unsigned int i = 1; i < n; i++) {
5571 + if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
5572 + loc_network_list_swap(list, i-1, i);
5578 + } while (swapped);
5581 +LOC_EXPORT int loc_network_list_merge(
5582 + struct loc_network_list* self, struct loc_network_list* other) {
5585 + for (unsigned int i = 0; i < other->size; i++) {
5586 + r = loc_network_list_push(self, other->list[i]);
5593 diff --git a/src/network.c b/src/network.c
5594 index d015579..28ca2df 100644
5598 #include <loc/compat.h>
5599 #include <loc/country.h>
5600 #include <loc/network.h>
5601 +#include <loc/network-list.h>
5602 #include <loc/private.h>
5604 struct loc_network {
5605 @@ -436,7 +437,7 @@ LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* othe
5609 -static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
5610 +LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
5611 // Families must match
5612 if (self->family != other->family)
5614 @@ -1106,207 +1107,3 @@ int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
5615 struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
5616 return loc_network_ref(node->network);
5621 -struct loc_network_list {
5622 - struct loc_ctx* ctx;
5625 - struct loc_network* list[1024];
5630 -LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
5631 - struct loc_network_list** list) {
5632 - struct loc_network_list* l = calloc(1, sizeof(*l));
5636 - l->ctx = loc_ref(ctx);
5639 - // Do not allow this list to grow larger than this
5640 - l->max_size = 1024;
5642 - DEBUG(l->ctx, "Network list allocated at %p\n", l);
5647 -LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
5653 -static void loc_network_list_free(struct loc_network_list* list) {
5654 - DEBUG(list->ctx, "Releasing network list at %p\n", list);
5656 - for (unsigned int i = 0; i < list->size; i++)
5657 - loc_network_unref(list->list[i]);
5659 - loc_unref(list->ctx);
5663 -LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
5667 - if (--list->refcount > 0)
5670 - loc_network_list_free(list);
5674 -LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
5675 - return list->size;
5678 -LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
5679 - return list->size == 0;
5682 -LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
5683 - for (unsigned int i = 0; i < list->size; i++)
5684 - loc_network_unref(list->list[i]);
5689 -LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
5690 - struct loc_network* network;
5693 - for (unsigned int i = 0; i < list->size; i++) {
5694 - network = list->list[i];
5696 - s = loc_network_str(network);
5698 - INFO(list->ctx, "%s\n", s);
5703 -LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
5705 - if (index >= list->size)
5708 - return loc_network_ref(list->list[index]);
5711 -LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
5712 - // Do not add networks that are already on the list
5713 - if (loc_network_list_contains(list, network))
5716 - // Check if we have space left
5717 - if (list->size == list->max_size) {
5718 - ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
5722 - DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
5724 - list->list[list->size++] = loc_network_ref(network);
5729 -LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
5730 - // Return nothing when empty
5731 - if (loc_network_list_empty(list)) {
5732 - DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5736 - struct loc_network* network = list->list[--list->size];
5738 - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5743 -LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
5744 - // Return nothing when empty
5745 - if (loc_network_list_empty(list)) {
5746 - DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5750 - struct loc_network* network = list->list[0];
5752 - // Move all elements to the top of the stack
5753 - for (unsigned int i = 0; i < --list->size; i++) {
5754 - list->list[i] = list->list[i+1];
5757 - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5762 -LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
5763 - for (unsigned int i = 0; i < list->size; i++) {
5764 - if (loc_network_eq(list->list[i], network))
5771 -static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
5772 - // Do nothing for invalid indices
5773 - if (i1 >= list->size || i2 >= list->size)
5776 - struct loc_network* network1 = list->list[i1];
5777 - struct loc_network* network2 = list->list[i2];
5779 - list->list[i1] = network2;
5780 - list->list[i2] = network1;
5783 -LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
5784 - unsigned int i = 0;
5785 - unsigned int j = list->size - 1;
5788 - loc_network_list_swap(list, i++, j--);
5792 -LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
5793 - unsigned int n = list->size;
5799 - for (unsigned int i = 1; i < n; i++) {
5800 - if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
5801 - loc_network_list_swap(list, i-1, i);
5807 - } while (swapped);
5810 -LOC_EXPORT int loc_network_list_merge(
5811 - struct loc_network_list* self, struct loc_network_list* other) {
5814 - for (unsigned int i = 0; i < other->size; i++) {
5815 - r = loc_network_list_push(self, other->list[i]);
5822 diff --git a/src/python/network.c b/src/python/network.c
5823 index 11f672b..ed91d65 100644
5824 --- a/src/python/network.c
5825 +++ b/src/python/network.c
5828 #include <loc/libloc.h>
5829 #include <loc/network.h>
5830 +#include <loc/network-list.h>
5832 #include "locationmodule.h"
5833 #include "network.h"
5837 From e646a8f35ec7eff009414b3fd107c9af5cf39a86 Mon Sep 17 00:00:00 2001
5838 From: Michael Tremer <michael.tremer@ipfire.org>
5839 Date: Mon, 16 Nov 2020 15:13:28 +0000
5840 Subject: [PATCH 056/111] Implement filtering for multiple countries in the
5843 This will allow us to speed up the export of the database
5844 if only a few countries should be returned.
5846 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
5849 src/country-list.c | 138 +++++++++++++++++++++++++++++++++++++++++
5851 src/database.c | 47 ++++++--------
5852 src/libloc.sym | 15 ++++-
5853 src/loc/country-list.h | 43 +++++++++++++
5854 src/loc/database.h | 5 +-
5855 src/python/database.c | 57 ++++++++++++++---
5856 8 files changed, 274 insertions(+), 36 deletions(-)
5857 create mode 100644 src/country-list.c
5858 create mode 100644 src/loc/country-list.h
5860 diff --git a/Makefile.am b/Makefile.am
5861 index f0d8c4c..f4ca3c8 100644
5864 @@ -93,6 +93,7 @@ pkginclude_HEADERS = \
5868 + src/loc/country-list.h \
5869 src/loc/database.h \
5872 @@ -109,6 +110,7 @@ src_libloc_la_SOURCES = \
5876 + src/country-list.c \
5879 src/network-list.c \
5880 diff --git a/src/country-list.c b/src/country-list.c
5881 new file mode 100644
5882 index 0000000..ae0d71a
5884 +++ b/src/country-list.c
5887 + libloc - A library to determine the location of someone on the Internet
5889 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
5891 + This library is free software; you can redistribute it and/or
5892 + modify it under the terms of the GNU Lesser General Public
5893 + License as published by the Free Software Foundation; either
5894 + version 2.1 of the License, or (at your option) any later version.
5896 + This library is distributed in the hope that it will be useful,
5897 + but WITHOUT ANY WARRANTY; without even the implied warranty of
5898 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5899 + Lesser General Public License for more details.
5903 +#include <stdlib.h>
5905 +#include <loc/country.h>
5906 +#include <loc/country-list.h>
5907 +#include <loc/private.h>
5909 +struct loc_country_list {
5910 + struct loc_ctx* ctx;
5913 + struct loc_country* list[1024];
5918 +LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
5919 + struct loc_country_list** list) {
5920 + struct loc_country_list* l = calloc(1, sizeof(*l));
5924 + l->ctx = loc_ref(ctx);
5927 + // Do not allow this list to grow larger than this
5928 + l->max_size = 1024;
5930 + DEBUG(l->ctx, "Country list allocated at %p\n", l);
5936 +LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
5942 +static void loc_country_list_free(struct loc_country_list* list) {
5943 + DEBUG(list->ctx, "Releasing country list at %p\n", list);
5945 + loc_country_list_clear(list);
5947 + loc_unref(list->ctx);
5951 +LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
5955 + if (--list->refcount > 0)
5958 + loc_country_list_free(list);
5962 +LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
5963 + return list->size;
5966 +LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
5967 + return list->size == 0;
5970 +LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
5971 + for (unsigned int i = 0; i < list->size; i++)
5972 + loc_country_unref(list->list[i]);
5975 +LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
5977 + if (index >= list->size)
5980 + return loc_country_ref(list->list[index]);
5983 +LOC_EXPORT int loc_country_list_append(
5984 + struct loc_country_list* list, struct loc_country* country) {
5985 + if (loc_country_list_contains(list, country))
5988 + // Check if we have space left
5989 + if (list->size == list->max_size) {
5990 + ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
5994 + DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
5996 + list->list[list->size++] = loc_country_ref(country);
6001 +LOC_EXPORT int loc_country_list_contains(
6002 + struct loc_country_list* list, struct loc_country* country) {
6003 + for (unsigned int i = 0; i < list->size; i++) {
6004 + if (loc_country_cmp(country, list->list[i]) == 0)
6011 +LOC_EXPORT int loc_country_list_contains_code(
6012 + struct loc_country_list* list, const char* code) {
6013 + struct loc_country* country;
6015 + int r = loc_country_new(list->ctx, &country, code);
6019 + r = loc_country_list_contains(list, country);
6020 + loc_country_unref(country);
6024 diff --git a/src/country.c b/src/country.c
6025 index 2ba93e6..7aac0db 100644
6028 @@ -34,6 +34,9 @@ struct loc_country {
6031 LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
6032 + if (!loc_country_code_is_valid(country_code))
6035 struct loc_country* c = calloc(1, sizeof(*c));
6038 diff --git a/src/database.c b/src/database.c
6039 index b9d870f..29823b2 100644
6040 --- a/src/database.c
6041 +++ b/src/database.c
6044 #include <loc/compat.h>
6045 #include <loc/country.h>
6046 +#include <loc/country-list.h>
6047 #include <loc/database.h>
6048 #include <loc/format.h>
6049 #include <loc/network.h>
6050 @@ -99,7 +100,7 @@ struct loc_database_enumerator {
6054 - char country_code[3];
6055 + struct loc_country_list* countries;
6057 enum loc_network_flags flags;
6059 @@ -1017,33 +1018,20 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
6063 -LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
6064 - // Set empty country code
6065 - if (!country_code || !*country_code) {
6066 - *enumerator->country_code = '\0';
6069 +LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
6070 + struct loc_database_enumerator* enumerator) {
6071 + if (!enumerator->countries)
6074 - // Treat A1, A2, A3 as special country codes,
6075 - // but perform search for flags instead
6076 - if (strcmp(country_code, "A1") == 0) {
6077 - return loc_database_enumerator_set_flag(enumerator,
6078 - LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
6079 - } else if (strcmp(country_code, "A2") == 0) {
6080 - return loc_database_enumerator_set_flag(enumerator,
6081 - LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
6082 - } else if (strcmp(country_code, "A3") == 0) {
6083 - return loc_database_enumerator_set_flag(enumerator,
6084 - LOC_NETWORK_FLAG_ANYCAST);
6086 + return loc_country_list_ref(enumerator->countries);
6089 - // Country codes must be two characters
6090 - if (!loc_country_code_is_valid(country_code))
6092 +LOC_EXPORT int loc_database_enumerator_set_countries(
6093 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
6094 + if (enumerator->countries)
6095 + loc_country_list_unref(enumerator->countries);
6097 - for (unsigned int i = 0; i < 3; i++) {
6098 - enumerator->country_code[i] = country_code[i];
6100 + enumerator->countries = loc_country_list_ref(countries);
6104 @@ -1129,6 +1117,12 @@ static int loc_database_enumerator_stack_push_node(
6108 +static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
6109 + const char* country_code = loc_network_get_country_code(network);
6111 + return loc_country_list_contains_code(countries, country_code);
6114 static int loc_database_enumerator_filter_network(
6115 struct loc_database_enumerator* enumerator, struct loc_network* network) {
6116 // Skip if the family does not match
6117 @@ -1136,8 +1130,7 @@ static int loc_database_enumerator_filter_network(
6120 // Skip if the country code does not match
6121 - if (*enumerator->country_code &&
6122 - !loc_network_match_country_code(network, enumerator->country_code))
6123 + if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
6126 // Skip if the ASN does not match
6127 diff --git a/src/libloc.sym b/src/libloc.sym
6128 index 453a1be..40e9f88 100644
6129 --- a/src/libloc.sym
6130 +++ b/src/libloc.sym
6131 @@ -49,6 +49,18 @@ global:
6132 loc_country_set_name;
6136 + loc_country_list_append;
6137 + loc_country_list_clear;
6138 + loc_country_list_contains;
6139 + loc_country_list_contains_code;
6140 + loc_country_list_empty;
6141 + loc_country_list_get;
6142 + loc_country_list_new;
6143 + loc_country_list_ref;
6144 + loc_country_list_size;
6145 + loc_country_list_unref;
6148 loc_database_add_as;
6149 loc_database_count_as;
6150 @@ -66,13 +78,14 @@ global:
6151 loc_database_verify;
6153 # Database Enumerator
6154 + loc_database_enumerator_get_countries;
6155 loc_database_enumerator_new;
6156 loc_database_enumerator_next_as;
6157 loc_database_enumerator_next_country;
6158 loc_database_enumerator_next_network;
6159 loc_database_enumerator_ref;
6160 loc_database_enumerator_set_asn;
6161 - loc_database_enumerator_set_country_code;
6162 + loc_database_enumerator_set_countries;
6163 loc_database_enumerator_set_family;
6164 loc_database_enumerator_set_flag;
6165 loc_database_enumerator_set_string;
6166 diff --git a/src/loc/country-list.h b/src/loc/country-list.h
6167 new file mode 100644
6168 index 0000000..a7f818a
6170 +++ b/src/loc/country-list.h
6173 + libloc - A library to determine the location of someone on the Internet
6175 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
6177 + This library is free software; you can redistribute it and/or
6178 + modify it under the terms of the GNU Lesser General Public
6179 + License as published by the Free Software Foundation; either
6180 + version 2.1 of the License, or (at your option) any later version.
6182 + This library is distributed in the hope that it will be useful,
6183 + but WITHOUT ANY WARRANTY; without even the implied warranty of
6184 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
6185 + Lesser General Public License for more details.
6188 +#ifndef LIBLOC_COUNTRY_LIST_H
6189 +#define LIBLOC_COUNTRY_LIST_H
6191 +#include <stdlib.h>
6193 +#include <loc/libloc.h>
6194 +#include <loc/country.h>
6196 +struct loc_country_list;
6198 +int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
6199 +struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
6200 +struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
6202 +size_t loc_country_list_size(struct loc_country_list* list);
6203 +int loc_country_list_empty(struct loc_country_list* list);
6204 +void loc_country_list_clear(struct loc_country_list* list);
6206 +struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
6207 +int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
6209 +int loc_country_list_contains(
6210 + struct loc_country_list* list, struct loc_country* country);
6211 +int loc_country_list_contains_code(
6212 + struct loc_country_list* list, const char* code);
6215 diff --git a/src/loc/database.h b/src/loc/database.h
6216 index 14eb5ea..246e5c5 100644
6217 --- a/src/loc/database.h
6218 +++ b/src/loc/database.h
6220 #include <loc/network.h>
6222 #include <loc/country.h>
6223 +#include <loc/country-list.h>
6225 struct loc_database;
6226 int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
6227 @@ -66,7 +67,9 @@ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_
6228 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
6230 int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
6231 -int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
6232 +struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
6233 +int loc_database_enumerator_set_countries(
6234 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
6235 int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
6236 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
6237 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
6238 diff --git a/src/python/database.c b/src/python/database.c
6239 index d169547..e6f6f37 100644
6240 --- a/src/python/database.c
6241 +++ b/src/python/database.c
6242 @@ -258,14 +258,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
6245 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
6246 - char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
6247 - const char* country_code = NULL;
6248 + char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
6249 + PyObject* country_codes = NULL;
6250 unsigned int asn = 0;
6255 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
6256 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
6257 + &PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
6260 struct loc_database_enumerator* enumerator;
6261 @@ -277,13 +278,55 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
6264 // Set country code we are searching for
6265 - if (country_code) {
6266 - r = loc_database_enumerator_set_country_code(enumerator, country_code);
6268 + if (country_codes) {
6269 + struct loc_country_list* countries;
6270 + r = loc_country_list_new(loc_ctx, &countries);
6272 - PyErr_SetFromErrno(PyExc_SystemError);
6273 + PyErr_SetString(PyExc_SystemError, "Could not create country list");
6277 + for (unsigned int i = 0; i < PyList_Size(country_codes); i++) {
6278 + PyObject* item = PyList_GetItem(country_codes, i);
6280 + if (!PyUnicode_Check(item)) {
6281 + PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
6282 + loc_country_list_unref(countries);
6286 + const char* country_code = PyUnicode_AsUTF8(item);
6288 + struct loc_country* country;
6289 + r = loc_country_new(loc_ctx, &country, country_code);
6291 + if (r == -EINVAL) {
6292 + PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
6294 + PyErr_SetString(PyExc_SystemError, "Could not create country");
6297 + loc_country_list_unref(countries);
6301 + // Append it to the list
6302 + r = loc_country_list_append(countries, country);
6304 + PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
6306 + loc_country_list_unref(countries);
6307 + loc_country_unref(country);
6311 + loc_country_unref(country);
6314 + loc_database_enumerator_set_countries(enumerator, countries);
6316 + Py_DECREF(country_codes);
6317 + loc_country_list_unref(countries);
6320 // Set the ASN we are searching for
6324 From 7af51f8a579c79714992a3e175036fb511139310 Mon Sep 17 00:00:00 2001
6325 From: Michael Tremer <michael.tremer@ipfire.org>
6326 Date: Mon, 16 Nov 2020 15:20:50 +0000
6327 Subject: [PATCH 057/111] python: Only return country codes we want
6329 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6331 src/python/export.py | 8 +++++++-
6332 1 file changed, 7 insertions(+), 1 deletion(-)
6334 diff --git a/src/python/export.py b/src/python/export.py
6335 index be4a68e..5e7fe53 100644
6336 --- a/src/python/export.py
6337 +++ b/src/python/export.py
6338 @@ -184,8 +184,14 @@ class Exporter(object):
6340 writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
6342 + # Filter countries from special country codes
6344 + country_code for country_code in countries if not country_code in flags.values()
6347 # Get all networks that match the family
6348 - networks = self.db.search_networks(family=family, flatten=True)
6349 + networks = self.db.search_networks(family=family,
6350 + country_codes=country_codes, flatten=True)
6352 # Walk through all networks
6353 for network in networks:
6357 From bd1dc6bf6fe4ce40bf12e7426e283b31afd274e1 Mon Sep 17 00:00:00 2001
6358 From: Michael Tremer <michael.tremer@ipfire.org>
6359 Date: Mon, 16 Nov 2020 15:25:15 +0000
6360 Subject: [PATCH 058/111] database: Filter flags in C
6362 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6364 src/python/export.py | 16 +++++++++++-----
6365 1 file changed, 11 insertions(+), 5 deletions(-)
6367 diff --git a/src/python/export.py b/src/python/export.py
6368 index 5e7fe53..739742f 100644
6369 --- a/src/python/export.py
6370 +++ b/src/python/export.py
6371 @@ -29,7 +29,7 @@ import _location
6372 log = logging.getLogger("location.export")
6377 _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
6378 _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
6379 _location.NETWORK_FLAG_ANYCAST : "A3",
6380 @@ -186,12 +186,18 @@ class Exporter(object):
6382 # Filter countries from special country codes
6384 - country_code for country_code in countries if not country_code in flags.values()
6385 + country_code for country_code in countries if not country_code in FLAGS.values()
6390 + for flag in FLAGS:
6391 + if FLAGS[flag] in countries:
6394 # Get all networks that match the family
6395 networks = self.db.search_networks(family=family,
6396 - country_codes=country_codes, flatten=True)
6397 + country_codes=country_codes, flags=flags, flatten=True)
6399 # Walk through all networks
6400 for network in networks:
6401 @@ -208,10 +214,10 @@ class Exporter(object):
6405 - for flag in flags:
6406 + for flag in FLAGS:
6407 if network.has_flag(flag):
6408 # Fetch the "fake" country code
6409 - country = flags[flag]
6410 + country = FLAGS[flag]
6413 writers[country].write(network)
6417 From 84a2f0c2d9cbf8ae4225802c29ccba86561c77ed Mon Sep 17 00:00:00 2001
6418 From: Michael Tremer <michael.tremer@ipfire.org>
6419 Date: Tue, 17 Nov 2020 16:46:48 +0000
6420 Subject: [PATCH 059/111] as: Add list for easier processing
6422 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6425 src/as-list.c | 138 ++++++++++++++++++++++++++++++++++++++++++
6426 src/database.c | 29 +++++++--
6427 src/libloc.sym | 15 ++++-
6428 src/loc/as-list.h | 41 +++++++++++++
6429 src/loc/database.h | 5 +-
6430 src/python/database.c | 58 ++++++++++++++++--
6431 src/python/export.py | 2 +-
6432 8 files changed, 275 insertions(+), 15 deletions(-)
6433 create mode 100644 src/as-list.c
6434 create mode 100644 src/loc/as-list.h
6436 diff --git a/Makefile.am b/Makefile.am
6437 index f4ca3c8..d0cc793 100644
6440 @@ -91,6 +91,7 @@ EXTRA_DIST += \
6441 pkginclude_HEADERS = \
6444 + src/loc/as-list.h \
6447 src/loc/country-list.h \
6448 @@ -109,6 +110,7 @@ lib_LTLIBRARIES = \
6449 src_libloc_la_SOURCES = \
6454 src/country-list.c \
6456 diff --git a/src/as-list.c b/src/as-list.c
6457 new file mode 100644
6458 index 0000000..7c69eb0
6463 + libloc - A library to determine the location of someone on the Internet
6465 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
6467 + This library is free software; you can redistribute it and/or
6468 + modify it under the terms of the GNU Lesser General Public
6469 + License as published by the Free Software Foundation; either
6470 + version 2.1 of the License, or (at your option) any later version.
6472 + This library is distributed in the hope that it will be useful,
6473 + but WITHOUT ANY WARRANTY; without even the implied warranty of
6474 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
6475 + Lesser General Public License for more details.
6479 +#include <stdlib.h>
6481 +#include <loc/as.h>
6482 +#include <loc/as-list.h>
6483 +#include <loc/private.h>
6485 +struct loc_as_list {
6486 + struct loc_ctx* ctx;
6489 + struct loc_as* list[1024];
6494 +LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
6495 + struct loc_as_list** list) {
6496 + struct loc_as_list* l = calloc(1, sizeof(*l));
6500 + l->ctx = loc_ref(ctx);
6503 + // Do not allow this list to grow larger than this
6504 + l->max_size = 1024;
6506 + DEBUG(l->ctx, "AS list allocated at %p\n", l);
6512 +LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
6518 +static void loc_as_list_free(struct loc_as_list* list) {
6519 + DEBUG(list->ctx, "Releasing AS list at %p\n", list);
6521 + loc_as_list_clear(list);
6523 + loc_unref(list->ctx);
6527 +LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
6531 + if (--list->refcount > 0)
6534 + loc_as_list_free(list);
6538 +LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
6539 + return list->size;
6542 +LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
6543 + return list->size == 0;
6546 +LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
6547 + for (unsigned int i = 0; i < list->size; i++)
6548 + loc_as_unref(list->list[i]);
6551 +LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
6553 + if (index >= list->size)
6556 + return loc_as_ref(list->list[index]);
6559 +LOC_EXPORT int loc_as_list_append(
6560 + struct loc_as_list* list, struct loc_as* as) {
6561 + if (loc_as_list_contains(list, as))
6564 + // Check if we have space left
6565 + if (list->size == list->max_size) {
6566 + ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
6570 + DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
6572 + list->list[list->size++] = loc_as_ref(as);
6577 +LOC_EXPORT int loc_as_list_contains(
6578 + struct loc_as_list* list, struct loc_as* as) {
6579 + for (unsigned int i = 0; i < list->size; i++) {
6580 + if (loc_as_cmp(as, list->list[i]) == 0)
6587 +LOC_EXPORT int loc_as_list_contains_number(
6588 + struct loc_as_list* list, uint32_t number) {
6589 + struct loc_as* as;
6591 + int r = loc_as_new(list->ctx, &as, number);
6595 + r = loc_as_list_contains(list, as);
6600 diff --git a/src/database.c b/src/database.c
6601 index 29823b2..51cb5cd 100644
6602 --- a/src/database.c
6603 +++ b/src/database.c
6606 #include <loc/libloc.h>
6608 +#include <loc/as-list.h>
6609 #include <loc/compat.h>
6610 #include <loc/country.h>
6611 #include <loc/country-list.h>
6612 @@ -101,7 +102,7 @@ struct loc_database_enumerator {
6615 struct loc_country_list* countries;
6617 + struct loc_as_list* asns;
6618 enum loc_network_flags flags;
6621 @@ -1036,9 +1037,20 @@ LOC_EXPORT int loc_database_enumerator_set_countries(
6625 -LOC_EXPORT int loc_database_enumerator_set_asn(
6626 - struct loc_database_enumerator* enumerator, unsigned int asn) {
6627 - enumerator->asn = asn;
6628 +LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
6629 + struct loc_database_enumerator* enumerator) {
6630 + if (!enumerator->asns)
6633 + return loc_as_list_ref(enumerator->asns);
6636 +LOC_EXPORT int loc_database_enumerator_set_asns(
6637 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
6638 + if (enumerator->asns)
6639 + loc_as_list_unref(enumerator->asns);
6641 + enumerator->asns = loc_as_list_ref(asns);
6645 @@ -1123,6 +1135,12 @@ static int loc_network_match_countries(struct loc_network* network, struct loc_c
6646 return loc_country_list_contains_code(countries, country_code);
6649 +static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
6650 + uint32_t asn = loc_network_get_asn(network);
6652 + return loc_as_list_contains_number(asns, asn);
6655 static int loc_database_enumerator_filter_network(
6656 struct loc_database_enumerator* enumerator, struct loc_network* network) {
6657 // Skip if the family does not match
6658 @@ -1134,8 +1152,7 @@ static int loc_database_enumerator_filter_network(
6661 // Skip if the ASN does not match
6662 - if (enumerator->asn &&
6663 - !loc_network_match_asn(network, enumerator->asn))
6664 + if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
6667 // Skip if flags do not match
6668 diff --git a/src/libloc.sym b/src/libloc.sym
6669 index 40e9f88..53273cd 100644
6670 --- a/src/libloc.sym
6671 +++ b/src/libloc.sym
6672 @@ -37,6 +37,18 @@ global:
6677 + loc_as_list_append;
6678 + loc_as_list_clear;
6679 + loc_as_list_contains;
6680 + loc_as_list_contains_number;
6681 + loc_as_list_empty;
6686 + loc_as_list_unref;
6690 loc_country_code_is_valid;
6691 @@ -78,13 +90,14 @@ global:
6692 loc_database_verify;
6694 # Database Enumerator
6695 + loc_database_enumerator_get_asns;
6696 loc_database_enumerator_get_countries;
6697 loc_database_enumerator_new;
6698 loc_database_enumerator_next_as;
6699 loc_database_enumerator_next_country;
6700 loc_database_enumerator_next_network;
6701 loc_database_enumerator_ref;
6702 - loc_database_enumerator_set_asn;
6703 + loc_database_enumerator_set_asns;
6704 loc_database_enumerator_set_countries;
6705 loc_database_enumerator_set_family;
6706 loc_database_enumerator_set_flag;
6707 diff --git a/src/loc/as-list.h b/src/loc/as-list.h
6708 new file mode 100644
6709 index 0000000..7b5c4e8
6711 +++ b/src/loc/as-list.h
6714 + libloc - A library to determine the location of someone on the Internet
6716 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
6718 + This library is free software; you can redistribute it and/or
6719 + modify it under the terms of the GNU Lesser General Public
6720 + License as published by the Free Software Foundation; either
6721 + version 2.1 of the License, or (at your option) any later version.
6723 + This library is distributed in the hope that it will be useful,
6724 + but WITHOUT ANY WARRANTY; without even the implied warranty of
6725 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
6726 + Lesser General Public License for more details.
6729 +#ifndef LIBLOC_AS_LIST_H
6730 +#define LIBLOC_AS_LIST_H
6732 +#include <loc/as.h>
6733 +#include <loc/libloc.h>
6735 +struct loc_as_list;
6737 +int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
6738 +struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
6739 +struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
6741 +size_t loc_as_list_size(struct loc_as_list* list);
6742 +int loc_as_list_empty(struct loc_as_list* list);
6743 +void loc_as_list_clear(struct loc_as_list* list);
6745 +struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
6746 +int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
6748 +int loc_as_list_contains(
6749 + struct loc_as_list* list, struct loc_as* as);
6750 +int loc_as_list_contains_number(
6751 + struct loc_as_list* list, uint32_t number);
6754 diff --git a/src/loc/database.h b/src/loc/database.h
6755 index 246e5c5..70801f0 100644
6756 --- a/src/loc/database.h
6757 +++ b/src/loc/database.h
6758 @@ -70,7 +70,10 @@ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerato
6759 struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
6760 int loc_database_enumerator_set_countries(
6761 struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
6762 -int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
6763 +struct loc_as_list* loc_database_enumerator_get_asns(
6764 + struct loc_database_enumerator* enumerator);
6765 +int loc_database_enumerator_set_asns(
6766 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
6767 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
6768 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
6769 int loc_database_enumerator_next_as(
6770 diff --git a/src/python/database.c b/src/python/database.c
6771 index e6f6f37..38a804c 100644
6772 --- a/src/python/database.c
6773 +++ b/src/python/database.c
6777 #include <loc/libloc.h>
6778 +#include <loc/as.h>
6779 +#include <loc/as-list.h>
6780 #include <loc/database.h>
6782 #include "locationmodule.h"
6783 @@ -258,15 +260,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
6786 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
6787 - char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
6788 + char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
6789 PyObject* country_codes = NULL;
6790 - unsigned int asn = 0;
6791 + PyObject* asn_list = NULL;
6796 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
6797 - &PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
6798 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
6799 + &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
6802 struct loc_database_enumerator* enumerator;
6803 @@ -330,13 +332,57 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
6806 // Set the ASN we are searching for
6808 - r = loc_database_enumerator_set_asn(enumerator, asn);
6810 + struct loc_as_list* asns;
6811 + r = loc_as_list_new(loc_ctx, &asns);
6813 + PyErr_SetString(PyExc_SystemError, "Could not create AS list");
6817 + for (unsigned int i = 0; i < PyList_Size(asn_list); i++) {
6818 + PyObject* item = PyList_GetItem(asn_list, i);
6820 + if (!PyLong_Check(item)) {
6821 + PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
6823 + loc_as_list_unref(asns);
6827 + unsigned long number = PyLong_AsLong(item);
6829 + struct loc_as* as;
6830 + r = loc_as_new(loc_ctx, &as, number);
6832 + PyErr_SetString(PyExc_SystemError, "Could not create AS");
6834 + loc_as_list_unref(asns);
6839 + r = loc_as_list_append(asns, as);
6841 + PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
6843 + loc_as_list_unref(asns);
6851 + r = loc_database_enumerator_set_asns(enumerator, asns);
6853 PyErr_SetFromErrno(PyExc_SystemError);
6855 + loc_as_list_unref(asns);
6859 + loc_as_list_unref(asns);
6862 // Set the flags we are searching for
6863 diff --git a/src/python/export.py b/src/python/export.py
6864 index 739742f..f675eb3 100644
6865 --- a/src/python/export.py
6866 +++ b/src/python/export.py
6867 @@ -197,7 +197,7 @@ class Exporter(object):
6869 # Get all networks that match the family
6870 networks = self.db.search_networks(family=family,
6871 - country_codes=country_codes, flags=flags, flatten=True)
6872 + country_codes=country_codes, asns=asns, flags=flags, flatten=True)
6874 # Walk through all networks
6875 for network in networks:
6879 From 50120b991fc2fa4b7813096de87b42d700faf3e6 Mon Sep 17 00:00:00 2001
6880 From: Michael Tremer <michael.tremer@ipfire.org>
6881 Date: Tue, 17 Nov 2020 16:56:43 +0000
6882 Subject: [PATCH 060/111] database: Simplify network matching code
6884 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6886 src/database.c | 16 ++++++++--------
6887 1 file changed, 8 insertions(+), 8 deletions(-)
6889 diff --git a/src/database.c b/src/database.c
6890 index 51cb5cd..1a354f6 100644
6891 --- a/src/database.c
6892 +++ b/src/database.c
6893 @@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
6897 -static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
6898 - const char* country_code = loc_network_get_country_code(network);
6900 - return loc_country_list_contains_code(countries, country_code);
6903 static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
6904 uint32_t asn = loc_network_get_asn(network);
6906 @@ -1148,8 +1142,14 @@ static int loc_database_enumerator_filter_network(
6909 // Skip if the country code does not match
6910 - if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
6912 + if (enumerator->countries) {
6913 + if (!loc_country_list_empty(enumerator->countries)) {
6914 + const char* country_code = loc_network_get_country_code(network);
6916 + if (!loc_country_list_contains_code(enumerator->countries, country_code))
6921 // Skip if the ASN does not match
6922 if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
6926 From c1a36c943181da5cd2aef589a972d5027e529eb8 Mon Sep 17 00:00:00 2001
6927 From: Michael Tremer <michael.tremer@ipfire.org>
6928 Date: Tue, 17 Nov 2020 16:58:55 +0000
6929 Subject: [PATCH 061/111] database: Simplify AS matching code
6931 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6933 src/database.c | 16 ++++++++--------
6934 1 file changed, 8 insertions(+), 8 deletions(-)
6936 diff --git a/src/database.c b/src/database.c
6937 index 1a354f6..be93e00 100644
6938 --- a/src/database.c
6939 +++ b/src/database.c
6940 @@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
6944 -static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
6945 - uint32_t asn = loc_network_get_asn(network);
6947 - return loc_as_list_contains_number(asns, asn);
6950 static int loc_database_enumerator_filter_network(
6951 struct loc_database_enumerator* enumerator, struct loc_network* network) {
6952 // Skip if the family does not match
6953 @@ -1152,8 +1146,14 @@ static int loc_database_enumerator_filter_network(
6956 // Skip if the ASN does not match
6957 - if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
6959 + if (enumerator->asns) {
6960 + if (!loc_as_list_empty(enumerator->asns)) {
6961 + uint32_t asn = loc_network_get_asn(network);
6963 + if (!loc_as_list_contains_number(enumerator->asns, asn))
6968 // Skip if flags do not match
6969 if (enumerator->flags &&
6973 From d5205091f9cc1ff987e483325d48696459df08d8 Mon Sep 17 00:00:00 2001
6974 From: Michael Tremer <michael.tremer@ipfire.org>
6975 Date: Tue, 17 Nov 2020 17:50:17 +0000
6976 Subject: [PATCH 062/111] countries: Make list grow dynamically
6978 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6980 src/country-list.c | 38 ++++++++++++++++++++++++++------------
6981 1 file changed, 26 insertions(+), 12 deletions(-)
6983 diff --git a/src/country-list.c b/src/country-list.c
6984 index ae0d71a..1ce2d06 100644
6985 --- a/src/country-list.c
6986 +++ b/src/country-list.c
6987 @@ -25,11 +25,27 @@ struct loc_country_list {
6988 struct loc_ctx* ctx;
6991 - struct loc_country* list[1024];
6992 + struct loc_country** elements;
6993 + size_t elements_size;
6999 +static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
7000 + DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
7001 + list, size, list->elements_size + size);
7003 + struct loc_country** elements = reallocarray(list->elements,
7004 + list->elements_size + size, sizeof(*list->elements));
7008 + list->elements = elements;
7009 + list->elements_size += size;
7014 LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
7015 struct loc_country_list** list) {
7016 struct loc_country_list* l = calloc(1, sizeof(*l));
7017 @@ -39,9 +55,6 @@ LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
7018 l->ctx = loc_ref(ctx);
7021 - // Do not allow this list to grow larger than this
7022 - l->max_size = 1024;
7024 DEBUG(l->ctx, "Country list allocated at %p\n", l);
7027 @@ -84,7 +97,7 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
7029 LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
7030 for (unsigned int i = 0; i < list->size; i++)
7031 - loc_country_unref(list->list[i]);
7032 + loc_country_unref(list->elements[i]);
7035 LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
7036 @@ -92,7 +105,7 @@ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* lis
7037 if (index >= list->size)
7040 - return loc_country_ref(list->list[index]);
7041 + return loc_country_ref(list->elements[index]);
7044 LOC_EXPORT int loc_country_list_append(
7045 @@ -101,14 +114,15 @@ LOC_EXPORT int loc_country_list_append(
7048 // Check if we have space left
7049 - if (list->size == list->max_size) {
7050 - ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
7052 + if (list->size >= list->elements_size) {
7053 + int r = loc_country_list_grow(list, 64);
7058 DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
7060 - list->list[list->size++] = loc_country_ref(country);
7061 + list->elements[list->size++] = loc_country_ref(country);
7065 @@ -116,7 +130,7 @@ LOC_EXPORT int loc_country_list_append(
7066 LOC_EXPORT int loc_country_list_contains(
7067 struct loc_country_list* list, struct loc_country* country) {
7068 for (unsigned int i = 0; i < list->size; i++) {
7069 - if (loc_country_cmp(country, list->list[i]) == 0)
7070 + if (loc_country_cmp(country, list->elements[i]) == 0)
7077 From 3b44e4211371d2103f89ba8f056b15edb7778fac Mon Sep 17 00:00:00 2001
7078 From: Michael Tremer <michael.tremer@ipfire.org>
7079 Date: Tue, 17 Nov 2020 17:55:51 +0000
7080 Subject: [PATCH 063/111] networks: Make list grow dynamically
7082 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7084 src/network-list.c | 60 ++++++++++++++++++++++++++++------------------
7085 1 file changed, 37 insertions(+), 23 deletions(-)
7087 diff --git a/src/network-list.c b/src/network-list.c
7088 index 1f6e80e..4912c02 100644
7089 --- a/src/network-list.c
7090 +++ b/src/network-list.c
7091 @@ -25,11 +25,27 @@ struct loc_network_list {
7092 struct loc_ctx* ctx;
7095 - struct loc_network* list[1024];
7096 + struct loc_network** elements;
7097 + size_t elements_size;
7103 +static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
7104 + DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
7105 + list, size, list->elements_size + size);
7107 + struct loc_network** elements = reallocarray(list->elements,
7108 + list->elements_size + size, sizeof(*list->elements));
7112 + list->elements = elements;
7113 + list->elements_size += size;
7118 LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
7119 struct loc_network_list** list) {
7120 struct loc_network_list* l = calloc(1, sizeof(*l));
7121 @@ -39,9 +55,6 @@ LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
7122 l->ctx = loc_ref(ctx);
7125 - // Do not allow this list to grow larger than this
7126 - l->max_size = 1024;
7128 DEBUG(l->ctx, "Network list allocated at %p\n", l);
7131 @@ -57,7 +70,7 @@ static void loc_network_list_free(struct loc_network_list* list) {
7132 DEBUG(list->ctx, "Releasing network list at %p\n", list);
7134 for (unsigned int i = 0; i < list->size; i++)
7135 - loc_network_unref(list->list[i]);
7136 + loc_network_unref(list->elements[i]);
7138 loc_unref(list->ctx);
7140 @@ -84,7 +97,7 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
7142 LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
7143 for (unsigned int i = 0; i < list->size; i++)
7144 - loc_network_unref(list->list[i]);
7145 + loc_network_unref(list->elements[i]);
7149 @@ -94,7 +107,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
7152 for (unsigned int i = 0; i < list->size; i++) {
7153 - network = list->list[i];
7154 + network = list->elements[i];
7156 s = loc_network_str(network);
7158 @@ -108,7 +121,7 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
7159 if (index >= list->size)
7162 - return loc_network_ref(list->list[index]);
7163 + return loc_network_ref(list->elements[index]);
7166 LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
7167 @@ -117,14 +130,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
7170 // Check if we have space left
7171 - if (list->size == list->max_size) {
7172 - ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
7174 + if (list->size >= list->elements_size) {
7175 + int r = loc_network_list_grow(list, 64);
7180 DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
7182 - list->list[list->size++] = loc_network_ref(network);
7183 + list->elements[list->size++] = loc_network_ref(network);
7187 @@ -136,7 +150,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
7191 - struct loc_network* network = list->list[--list->size];
7192 + struct loc_network* network = list->elements[--list->size];
7194 DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
7196 @@ -150,11 +164,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
7200 - struct loc_network* network = list->list[0];
7201 + struct loc_network* network = list->elements[0];
7203 // Move all elements to the top of the stack
7204 for (unsigned int i = 0; i < --list->size; i++) {
7205 - list->list[i] = list->list[i+1];
7206 + list->elements[i] = list->elements[i+1];
7209 DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
7210 @@ -164,7 +178,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
7212 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
7213 for (unsigned int i = 0; i < list->size; i++) {
7214 - if (loc_network_eq(list->list[i], network))
7215 + if (loc_network_eq(list->elements[i], network))
7219 @@ -176,11 +190,11 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
7220 if (i1 >= list->size || i2 >= list->size)
7223 - struct loc_network* network1 = list->list[i1];
7224 - struct loc_network* network2 = list->list[i2];
7225 + struct loc_network* network1 = list->elements[i1];
7226 + struct loc_network* network2 = list->elements[i2];
7228 - list->list[i1] = network2;
7229 - list->list[i2] = network1;
7230 + list->elements[i1] = network2;
7231 + list->elements[i2] = network1;
7234 LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
7235 @@ -200,7 +214,7 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
7238 for (unsigned int i = 1; i < n; i++) {
7239 - if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
7240 + if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) {
7241 loc_network_list_swap(list, i-1, i);
7244 @@ -215,7 +229,7 @@ LOC_EXPORT int loc_network_list_merge(
7247 for (unsigned int i = 0; i < other->size; i++) {
7248 - r = loc_network_list_push(self, other->list[i]);
7249 + r = loc_network_list_push(self, other->elements[i]);
7256 From 1a415f8c555f4fe9a68eb2a897c4a1fc0d33db25 Mon Sep 17 00:00:00 2001
7257 From: Michael Tremer <michael.tremer@ipfire.org>
7258 Date: Tue, 17 Nov 2020 17:57:55 +0000
7259 Subject: [PATCH 064/111] as: Make lists grow dynamically
7261 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7263 src/as-list.c | 38 ++++++++++++++++++++++++++------------
7264 1 file changed, 26 insertions(+), 12 deletions(-)
7266 diff --git a/src/as-list.c b/src/as-list.c
7267 index 7c69eb0..17de23e 100644
7270 @@ -25,11 +25,27 @@ struct loc_as_list {
7271 struct loc_ctx* ctx;
7274 - struct loc_as* list[1024];
7275 + struct loc_as** elements;
7276 + size_t elements_size;
7282 +static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
7283 + DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
7284 + list, size, list->elements_size + size);
7286 + struct loc_as** elements = reallocarray(list->elements,
7287 + list->elements_size + size, sizeof(*list->elements));
7291 + list->elements = elements;
7292 + list->elements_size += size;
7297 LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
7298 struct loc_as_list** list) {
7299 struct loc_as_list* l = calloc(1, sizeof(*l));
7300 @@ -39,9 +55,6 @@ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
7301 l->ctx = loc_ref(ctx);
7304 - // Do not allow this list to grow larger than this
7305 - l->max_size = 1024;
7307 DEBUG(l->ctx, "AS list allocated at %p\n", l);
7310 @@ -84,7 +97,7 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
7312 LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
7313 for (unsigned int i = 0; i < list->size; i++)
7314 - loc_as_unref(list->list[i]);
7315 + loc_as_unref(list->elements[i]);
7318 LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
7319 @@ -92,7 +105,7 @@ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index
7320 if (index >= list->size)
7323 - return loc_as_ref(list->list[index]);
7324 + return loc_as_ref(list->elements[index]);
7327 LOC_EXPORT int loc_as_list_append(
7328 @@ -101,14 +114,15 @@ LOC_EXPORT int loc_as_list_append(
7331 // Check if we have space left
7332 - if (list->size == list->max_size) {
7333 - ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
7335 + if (list->size >= list->elements_size) {
7336 + int r = loc_as_list_grow(list, 64);
7341 DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
7343 - list->list[list->size++] = loc_as_ref(as);
7344 + list->elements[list->size++] = loc_as_ref(as);
7348 @@ -116,7 +130,7 @@ LOC_EXPORT int loc_as_list_append(
7349 LOC_EXPORT int loc_as_list_contains(
7350 struct loc_as_list* list, struct loc_as* as) {
7351 for (unsigned int i = 0; i < list->size; i++) {
7352 - if (loc_as_cmp(as, list->list[i]) == 0)
7353 + if (loc_as_cmp(as, list->elements[i]) == 0)
7360 From e6592434ee7836507c1f436ec3b0db3bc81a81b9 Mon Sep 17 00:00:00 2001
7361 From: Michael Tremer <michael.tremer@ipfire.org>
7362 Date: Tue, 17 Nov 2020 18:13:49 +0000
7363 Subject: [PATCH 065/111] export: Change back to use Network objects
7365 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7367 src/python/export.py | 4 +---
7368 1 file changed, 1 insertion(+), 3 deletions(-)
7370 diff --git a/src/python/export.py b/src/python/export.py
7371 index f675eb3..67e437f 100644
7372 --- a/src/python/export.py
7373 +++ b/src/python/export.py
7374 @@ -144,9 +144,7 @@ class XTGeoIPOutputWriter(OutputWriter):
7375 def _write_network(self, network):
7376 for address in (network.first_address, network.last_address):
7377 # Convert this into a string of bits
7378 - bytes = socket.inet_pton(
7379 - socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
7381 + bytes = socket.inet_pton(network.family, address)
7388 From 248f5e0419f2349253b8ea96e477c15649fe2173 Mon Sep 17 00:00:00 2001
7389 From: Michael Tremer <michael.tremer@ipfire.org>
7390 Date: Tue, 17 Nov 2020 18:14:15 +0000
7391 Subject: [PATCH 066/111] Actually clear all lists
7393 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7395 src/as-list.c | 8 ++++++++
7396 src/country-list.c | 8 ++++++++
7397 src/network-list.c | 6 ++++++
7398 3 files changed, 22 insertions(+)
7400 diff --git a/src/as-list.c b/src/as-list.c
7401 index 17de23e..76620c7 100644
7404 @@ -96,8 +96,16 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
7407 LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
7408 + if (!list->elements)
7411 for (unsigned int i = 0; i < list->size; i++)
7412 loc_as_unref(list->elements[i]);
7414 + free(list->elements);
7415 + list->elements_size = 0;
7420 LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
7421 diff --git a/src/country-list.c b/src/country-list.c
7422 index 1ce2d06..1c49c47 100644
7423 --- a/src/country-list.c
7424 +++ b/src/country-list.c
7425 @@ -96,8 +96,16 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
7428 LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
7429 + if (!list->elements)
7432 for (unsigned int i = 0; i < list->size; i++)
7433 loc_country_unref(list->elements[i]);
7435 + free(list->elements);
7436 + list->elements_size = 0;
7441 LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
7442 diff --git a/src/network-list.c b/src/network-list.c
7443 index 4912c02..9cb4547 100644
7444 --- a/src/network-list.c
7445 +++ b/src/network-list.c
7446 @@ -96,9 +96,15 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
7449 LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
7450 + if (!list->elements)
7453 for (unsigned int i = 0; i < list->size; i++)
7454 loc_network_unref(list->elements[i]);
7456 + free(list->elements);
7457 + list->elements_size = 0;
7465 From c98ebf8aae2aa141193db52cd9429b1ded5b09c4 Mon Sep 17 00:00:00 2001
7466 From: Michael Tremer <michael.tremer@ipfire.org>
7467 Date: Tue, 17 Nov 2020 18:34:51 +0000
7468 Subject: [PATCH 067/111] database: Do not clean up python list
7470 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7472 src/python/database.c | 9 +++++++--
7473 1 file changed, 7 insertions(+), 2 deletions(-)
7475 diff --git a/src/python/database.c b/src/python/database.c
7476 index 38a804c..ed22275 100644
7477 --- a/src/python/database.c
7478 +++ b/src/python/database.c
7479 @@ -325,9 +325,14 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
7480 loc_country_unref(country);
7483 - loc_database_enumerator_set_countries(enumerator, countries);
7484 + r = loc_database_enumerator_set_countries(enumerator, countries);
7486 + PyErr_SetFromErrno(PyExc_SystemError);
7488 + loc_as_list_unref(countries);
7492 - Py_DECREF(country_codes);
7493 loc_country_list_unref(countries);
7499 From 5470d06cb59027f4e04b6d576763dbf7f1093fde Mon Sep 17 00:00:00 2001
7500 From: Michael Tremer <michael.tremer@ipfire.org>
7501 Date: Tue, 17 Nov 2020 19:01:04 +0000
7502 Subject: [PATCH 068/111] database: Free filter lists in enumerator
7504 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7506 src/database.c | 6 ++++++
7507 1 file changed, 6 insertions(+)
7509 diff --git a/src/database.c b/src/database.c
7510 index be93e00..ca35fe1 100644
7511 --- a/src/database.c
7512 +++ b/src/database.c
7513 @@ -950,6 +950,12 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera
7514 if (enumerator->string)
7515 free(enumerator->string);
7517 + if (enumerator->countries)
7518 + loc_country_list_unref(enumerator->countries);
7520 + if (enumerator->asns)
7521 + loc_as_list_unref(enumerator->asns);
7523 // Free network search
7524 free(enumerator->networks_visited);
7529 From e0e96878d3df51c4a265d51d088005dedf9335e3 Mon Sep 17 00:00:00 2001
7530 From: Michael Tremer <michael.tremer@ipfire.org>
7531 Date: Wed, 18 Nov 2020 13:18:52 +0000
7532 Subject: [PATCH 069/111] database: Add debug output to filtering
7534 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7536 src/database.c | 29 ++++++++++++++++-------------
7537 1 file changed, 16 insertions(+), 13 deletions(-)
7539 diff --git a/src/database.c b/src/database.c
7540 index ca35fe1..83dd752 100644
7541 --- a/src/database.c
7542 +++ b/src/database.c
7543 @@ -1138,33 +1138,36 @@ static int loc_database_enumerator_stack_push_node(
7544 static int loc_database_enumerator_filter_network(
7545 struct loc_database_enumerator* enumerator, struct loc_network* network) {
7546 // Skip if the family does not match
7547 - if (enumerator->family && loc_network_address_family(network) != enumerator->family)
7548 + if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
7549 + DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
7553 // Skip if the country code does not match
7554 - if (enumerator->countries) {
7555 - if (!loc_country_list_empty(enumerator->countries)) {
7556 - const char* country_code = loc_network_get_country_code(network);
7557 + if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
7558 + const char* country_code = loc_network_get_country_code(network);
7560 - if (!loc_country_list_contains_code(enumerator->countries, country_code))
7562 + if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
7563 + DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
7568 // Skip if the ASN does not match
7569 - if (enumerator->asns) {
7570 - if (!loc_as_list_empty(enumerator->asns)) {
7571 - uint32_t asn = loc_network_get_asn(network);
7572 + if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
7573 + uint32_t asn = loc_network_get_asn(network);
7575 - if (!loc_as_list_contains_number(enumerator->asns, asn))
7577 + if (!loc_as_list_contains_number(enumerator->asns, asn)) {
7578 + DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
7583 // Skip if flags do not match
7584 - if (enumerator->flags &&
7585 - !loc_network_match_flag(network, enumerator->flags))
7586 + if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
7587 + DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
7596 From bce0c9295ff8ff9488f24babe01ce851228d0b1e Mon Sep 17 00:00:00 2001
7597 From: Michael Tremer <michael.tremer@ipfire.org>
7598 Date: Wed, 18 Nov 2020 13:19:04 +0000
7599 Subject: [PATCH 070/111] export: Remove filtering for flags
7601 The filter is an AND filter and if we set the flags from
7602 the special country codes, we won't get back much.
7604 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7606 src/python/export.py | 8 +-------
7607 1 file changed, 1 insertion(+), 7 deletions(-)
7609 diff --git a/src/python/export.py b/src/python/export.py
7610 index 67e437f..4219957 100644
7611 --- a/src/python/export.py
7612 +++ b/src/python/export.py
7613 @@ -187,15 +187,9 @@ class Exporter(object):
7614 country_code for country_code in countries if not country_code in FLAGS.values()
7619 - for flag in FLAGS:
7620 - if FLAGS[flag] in countries:
7623 # Get all networks that match the family
7624 networks = self.db.search_networks(family=family,
7625 - country_codes=country_codes, asns=asns, flags=flags, flatten=True)
7626 + country_codes=country_codes, asns=asns, flatten=True)
7628 # Walk through all networks
7629 for network in networks:
7633 From 627bf1daaae1510cfd4016297ed16b82df209aae Mon Sep 17 00:00:00 2001
7634 From: Michael Tremer <michael.tremer@ipfire.org>
7635 Date: Wed, 18 Nov 2020 13:33:45 +0000
7636 Subject: [PATCH 071/111] python: Remove unnecessary db object from writers
7638 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7640 src/python/export.py | 12 ++++++------
7641 1 file changed, 6 insertions(+), 6 deletions(-)
7643 diff --git a/src/python/export.py b/src/python/export.py
7644 index 4219957..5bc9f30 100644
7645 --- a/src/python/export.py
7646 +++ b/src/python/export.py
7647 @@ -39,8 +39,8 @@ class OutputWriter(object):
7651 - def __init__(self, db, f, prefix=None, flatten=True):
7652 - self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten
7653 + def __init__(self, f, prefix=None, flatten=True):
7654 + self.f, self.prefix, self.flatten = f, prefix, flatten
7656 # The previously written network
7657 self._last_network = None
7658 @@ -49,13 +49,13 @@ class OutputWriter(object):
7659 self._write_header()
7662 - def open(cls, db, filename, **kwargs):
7663 + def open(cls, filename, **kwargs):
7665 Convenience function to open a file
7667 f = open(filename, cls.mode)
7669 - return cls(db, f, **kwargs)
7670 + return cls(f, **kwargs)
7673 return "<%s f=%s>" % (self.__class__.__name__, self.f)
7674 @@ -172,7 +172,7 @@ class Exporter(object):
7675 directory, prefix=country_code, suffix=self.writer.suffix, family=family,
7678 - writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code)
7679 + writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
7681 # Create writers for ASNs
7683 @@ -180,7 +180,7 @@ class Exporter(object):
7684 directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
7687 - writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
7688 + writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
7690 # Filter countries from special country codes
7695 From 9cb56ac9adafafa6e452009c2fa2d42e94474e11 Mon Sep 17 00:00:00 2001
7696 From: Michael Tremer <michael.tremer@ipfire.org>
7697 Date: Wed, 18 Nov 2020 13:34:50 +0000
7698 Subject: [PATCH 072/111] location: End lookup after an invalid IP address was
7701 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7703 src/python/location.in | 1 +
7704 1 file changed, 1 insertion(+)
7706 diff --git a/src/python/location.in b/src/python/location.in
7707 index 0d09210..6885ea0 100644
7708 --- a/src/python/location.in
7709 +++ b/src/python/location.in
7710 @@ -253,6 +253,7 @@ class CLI(object):
7711 network = db.lookup(address)
7713 print(_("Invalid IP address: %s") % address, file=sys.stderr)
7717 "address" : address,
7721 From 2a550d12208f8bc8002e05ac08613312df26b20e Mon Sep 17 00:00:00 2001
7722 From: Michael Tremer <michael.tremer@ipfire.org>
7723 Date: Thu, 19 Nov 2020 12:03:33 +0000
7724 Subject: [PATCH 073/111] python: Fix download of database
7726 This was all messed up in 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1
7727 when the change of type for the timestamp wasn't changed everywhere
7730 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7732 src/python/downloader.py | 6 +++---
7733 src/python/location.in | 2 +-
7734 2 files changed, 4 insertions(+), 4 deletions(-)
7736 diff --git a/src/python/downloader.py b/src/python/downloader.py
7737 index 87bbb68..05f7872 100644
7738 --- a/src/python/downloader.py
7739 +++ b/src/python/downloader.py
7740 @@ -119,8 +119,8 @@ class Downloader(object):
7744 - headers["If-Modified-Since"] = timestamp.strftime(
7745 - "%a, %d %b %Y %H:%M:%S GMT",
7746 + headers["If-Modified-Since"] = time.strftime(
7747 + "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
7750 t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
7751 @@ -195,7 +195,7 @@ class Downloader(object):
7752 db = Database(f.name)
7754 # Database is not recent
7755 - if timestamp and db.created_at < timestamp.timestamp():
7756 + if timestamp and db.created_at < timestamp:
7759 log.info("Downloaded new database from %s" % (time.strftime(
7760 diff --git a/src/python/location.in b/src/python/location.in
7761 index 6885ea0..b30beae 100644
7762 --- a/src/python/location.in
7763 +++ b/src/python/location.in
7764 @@ -433,7 +433,7 @@ class CLI(object):
7766 # Try downloading a new database
7768 - t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
7769 + t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
7771 # If no file could be downloaded, log a message
7772 except FileNotFoundError as e:
7776 From a1a00053300cff3c0f690d52377c76c83c2a08b2 Mon Sep 17 00:00:00 2001
7777 From: Michael Tremer <michael.tremer@ipfire.org>
7778 Date: Thu, 19 Nov 2020 12:34:11 +0000
7779 Subject: [PATCH 074/111] python: Add property to return IP addresses as bytes
7781 This avoids calling inet_pton to parse IP addresses from string
7783 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7785 src/libloc.sym | 2 ++
7786 src/loc/network.h | 2 ++
7787 src/network.c | 8 ++++++++
7788 src/python/database.c | 2 +-
7789 src/python/export.py | 7 ++-----
7790 src/python/network.c | 40 ++++++++++++++++++++++++++++++++++++++++
7791 6 files changed, 55 insertions(+), 6 deletions(-)
7793 diff --git a/src/libloc.sym b/src/libloc.sym
7794 index 53273cd..406dd15 100644
7795 --- a/src/libloc.sym
7796 +++ b/src/libloc.sym
7797 @@ -113,6 +113,8 @@ global:
7798 loc_network_format_last_address;
7799 loc_network_get_asn;
7800 loc_network_get_country_code;
7801 + loc_network_get_first_address;
7802 + loc_network_get_last_address;
7804 loc_network_has_flag;
7805 loc_network_is_subnet;
7806 diff --git a/src/loc/network.h b/src/loc/network.h
7807 index d86b685..4b7410c 100644
7808 --- a/src/loc/network.h
7809 +++ b/src/loc/network.h
7810 @@ -39,7 +39,9 @@ struct loc_network* loc_network_unref(struct loc_network* network);
7811 char* loc_network_str(struct loc_network* network);
7812 int loc_network_address_family(struct loc_network* network);
7814 +const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
7815 char* loc_network_format_first_address(struct loc_network* network);
7816 +const struct in6_addr* loc_network_get_last_address(struct loc_network* network);
7817 char* loc_network_format_last_address(struct loc_network* network);
7818 int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
7820 diff --git a/src/network.c b/src/network.c
7821 index 28ca2df..4c8787a 100644
7824 @@ -343,10 +343,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc
7828 +LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) {
7829 + return &network->first_address;
7832 LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) {
7833 return loc_network_format_address(network, &network->first_address);
7836 +LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) {
7837 + return &network->last_address;
7840 LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
7841 return loc_network_format_address(network, &network->last_address);
7843 diff --git a/src/python/database.c b/src/python/database.c
7844 index ed22275..f385c61 100644
7845 --- a/src/python/database.c
7846 +++ b/src/python/database.c
7847 @@ -329,7 +329,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
7849 PyErr_SetFromErrno(PyExc_SystemError);
7851 - loc_as_list_unref(countries);
7852 + loc_country_list_unref(countries);
7856 diff --git a/src/python/export.py b/src/python/export.py
7857 index 5bc9f30..6b39878 100644
7858 --- a/src/python/export.py
7859 +++ b/src/python/export.py
7860 @@ -142,11 +142,8 @@ class XTGeoIPOutputWriter(OutputWriter):
7863 def _write_network(self, network):
7864 - for address in (network.first_address, network.last_address):
7865 - # Convert this into a string of bits
7866 - bytes = socket.inet_pton(network.family, address)
7868 - self.f.write(bytes)
7869 + for address in (network._first_address, network._last_address):
7870 + self.f.write(address)
7874 diff --git a/src/python/network.c b/src/python/network.c
7875 index ed91d65..742b472 100644
7876 --- a/src/python/network.c
7877 +++ b/src/python/network.c
7878 @@ -215,6 +215,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) {
7882 +static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) {
7883 + struct in_addr address4;
7885 + // Convert IPv4 addresses to struct in_addr
7886 + if (IN6_IS_ADDR_V4MAPPED(address6)) {
7887 + address4.s_addr = address6->s6_addr32[3];
7889 + return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4));
7892 + // Return IPv6 addresses as they are
7893 + return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6));
7896 +static PyObject* Network_get__first_address(NetworkObject* self) {
7897 + const struct in6_addr* address = loc_network_get_first_address(self->network);
7899 + return PyBytes_FromAddress(address);
7902 static PyObject* Network_get_last_address(NetworkObject* self) {
7903 char* address = loc_network_format_last_address(self->network);
7905 @@ -224,6 +244,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
7909 +static PyObject* Network_get__last_address(NetworkObject* self) {
7910 + const struct in6_addr* address = loc_network_get_last_address(self->network);
7912 + return PyBytes_FromAddress(address);
7915 static struct PyMethodDef Network_methods[] = {
7918 @@ -281,6 +307,13 @@ static struct PyGetSetDef Network_getsetters[] = {
7924 + (getter)Network_get__first_address,
7931 (getter)Network_get_last_address,
7932 @@ -288,6 +321,13 @@ static struct PyGetSetDef Network_getsetters[] = {
7938 + (getter)Network_get__last_address,
7949 From 90d2194a876c223f9124ce9e27bdee6a6b49ff6a Mon Sep 17 00:00:00 2001
7950 From: Michael Tremer <michael.tremer@ipfire.org>
7951 Date: Thu, 19 Nov 2020 12:40:01 +0000
7952 Subject: [PATCH 075/111] export: Remove old flattening feature
7954 The database enumerator now only returns networks that will
7957 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7959 src/python/export.py | 34 ++++++----------------------------
7960 1 file changed, 6 insertions(+), 28 deletions(-)
7962 diff --git a/src/python/export.py b/src/python/export.py
7963 index 6b39878..4702bcf 100644
7964 --- a/src/python/export.py
7965 +++ b/src/python/export.py
7966 @@ -39,11 +39,8 @@ class OutputWriter(object):
7970 - def __init__(self, f, prefix=None, flatten=True):
7971 - self.f, self.prefix, self.flatten = f, prefix, flatten
7973 - # The previously written network
7974 - self._last_network = None
7975 + def __init__(self, f, prefix=None):
7976 + self.f, self.prefix = f, prefix
7978 # Immediately write the header
7979 self._write_header()
7980 @@ -60,18 +57,6 @@ class OutputWriter(object):
7982 return "<%s f=%s>" % (self.__class__.__name__, self.f)
7984 - def _flatten(self, network):
7986 - Checks if the given network needs to be written to file,
7987 - or if it is a subnet of the previously written network.
7989 - if self._last_network and network.is_subnet_of(self._last_network):
7992 - # Remember this network for the next call
7993 - self._last_network = network
7996 def _write_header(self):
7998 The header of the file
7999 @@ -84,15 +69,8 @@ class OutputWriter(object):
8003 - def _write_network(self, network):
8004 - self.f.write("%s\n" % network)
8006 def write(self, network):
8007 - if self.flatten and self._flatten(network):
8008 - log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
8011 - return self._write_network(network)
8012 + self.f.write("%s\n" % network)
8016 @@ -113,7 +91,7 @@ class IpsetOutputWriter(OutputWriter):
8017 def _write_header(self):
8018 self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
8020 - def _write_network(self, network):
8021 + def write(self, network):
8022 self.f.write("add %s %s\n" % (self.prefix, network))
8025 @@ -129,7 +107,7 @@ class NftablesOutputWriter(OutputWriter):
8026 def _write_footer(self):
8029 - def _write_network(self, network):
8030 + def write(self, network):
8031 self.f.write(" %s,\n" % network)
8034 @@ -141,7 +119,7 @@ class XTGeoIPOutputWriter(OutputWriter):
8038 - def _write_network(self, network):
8039 + def write(self, network):
8040 for address in (network._first_address, network._last_address):
8041 self.f.write(address)
8046 From 90188dad86223380b0854b523b63ec024117c5f2 Mon Sep 17 00:00:00 2001
8047 From: Michael Tremer <michael.tremer@ipfire.org>
8048 Date: Thu, 19 Nov 2020 12:41:19 +0000
8049 Subject: [PATCH 076/111] export: Speed-up export in xt_geoip format
8051 Removing the loop avoids creating a tuple and then iterating over it
8053 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
8055 src/python/export.py | 4 ++--
8056 1 file changed, 2 insertions(+), 2 deletions(-)
8058 diff --git a/src/python/export.py b/src/python/export.py
8059 index 4702bcf..f0eae26 100644
8060 --- a/src/python/export.py
8061 +++ b/src/python/export.py
8062 @@ -120,8 +120,8 @@ class XTGeoIPOutputWriter(OutputWriter):
8065 def write(self, network):
8066 - for address in (network._first_address, network._last_address):
8067 - self.f.write(address)
8068 + self.f.write(network._first_address)
8069 + self.f.write(network._last_address)
8076 From 6661692f3bc8f788af8b75ae25561f4cc4a2acb5 Mon Sep 17 00:00:00 2001
8077 From: Michael Tremer <michael.tremer@ipfire.org>
8078 Date: Thu, 19 Nov 2020 12:48:46 +0000
8079 Subject: [PATCH 077/111] database: Disable some useless code when not running
8082 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
8084 src/database.c | 16 +++++++++++++++-
8085 1 file changed, 15 insertions(+), 1 deletion(-)
8087 diff --git a/src/database.c b/src/database.c
8088 index 83dd752..ef4f505 100644
8089 --- a/src/database.c
8090 +++ b/src/database.c
8091 @@ -619,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
8094 clock_t end = clock();
8095 - DEBUG(db->ctx, "Signature checked in %.4fms\n",
8096 + INFO(db->ctx, "Signature checked in %.4fms\n",
8097 (double)(end - start) / CLOCKS_PER_SEC * 1000);
8100 @@ -679,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
8102 off_t hi = db->as_count - 1;
8104 +#ifdef ENABLE_DEBUG
8106 clock_t start = clock();
8110 off_t i = (lo + hi) / 2;
8111 @@ -693,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
8112 // Check if this is a match
8113 uint32_t as_number = loc_as_get_number(*as);
8114 if (as_number == number) {
8115 +#ifdef ENABLE_DEBUG
8116 clock_t end = clock();
8118 // Log how fast this has been
8119 DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
8120 (double)(end - start) / CLOCKS_PER_SEC * 1000);
8125 @@ -741,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
8129 +#ifdef ENABLE_DEBUG
8131 char* string = loc_network_str(*network);
8132 DEBUG(db->ctx, "Got network %s\n", string);
8139 @@ -840,17 +846,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
8143 +#ifdef ENABLE_DEBUG
8145 clock_t start = clock();
8148 int r = __loc_database_lookup(db, address, network, &network_address,
8149 db->network_nodes_v1, 0);
8151 +#ifdef ENABLE_DEBUG
8152 clock_t end = clock();
8154 // Log how fast this has been
8155 DEBUG(db->ctx, "Executed network search in %.4fms\n",
8156 (double)(end - start) / CLOCKS_PER_SEC * 1000);
8161 @@ -897,8 +907,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
8163 off_t hi = db->countries_count - 1;
8165 +#ifdef ENABLE_DEBUG
8167 clock_t start = clock();
8171 off_t i = (lo + hi) / 2;
8172 @@ -913,11 +925,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
8173 int result = strcmp(code, cc);
8176 +#ifdef ENABLE_DEBUG
8177 clock_t end = clock();
8179 // Log how fast this has been
8180 DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
8181 (double)(end - start) / CLOCKS_PER_SEC * 1000);
8189 From a17c353bce55a9f38f67b4b7d2425194cfa208e7 Mon Sep 17 00:00:00 2001
8190 From: Michael Tremer <michael.tremer@ipfire.org>
8191 Date: Fri, 20 Nov 2020 14:43:31 +0000
8192 Subject: [PATCH 078/111] test: Add tests to network-lists
8194 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
8196 Makefile.am | 10 ++++
8197 src/.gitignore | 1 +
8198 src/test-network-list.c | 126 ++++++++++++++++++++++++++++++++++++++++
8199 3 files changed, 137 insertions(+)
8200 create mode 100644 src/test-network-list.c
8202 diff --git a/Makefile.am b/Makefile.am
8203 index d0cc793..ebd7e17 100644
8206 @@ -318,6 +318,7 @@ check_PROGRAMS = \
8210 + src/test-network-list \
8214 @@ -357,6 +358,15 @@ src_test_network_CFLAGS = \
8215 src_test_network_LDADD = \
8218 +src_test_network_list_SOURCES = \
8219 + src/test-network-list.c
8221 +src_test_network_list_CFLAGS = \
8224 +src_test_network_list_LDADD = \
8227 src_test_stringpool_SOURCES = \
8228 src/test-stringpool.c
8230 diff --git a/src/.gitignore b/src/.gitignore
8231 index caf80b5..3ccbdb8 100644
8232 --- a/src/.gitignore
8233 +++ b/src/.gitignore
8234 @@ -10,5 +10,6 @@ test-libloc
8241 diff --git a/src/test-network-list.c b/src/test-network-list.c
8242 new file mode 100644
8243 index 0000000..3061d63
8245 +++ b/src/test-network-list.c
8248 + libloc - A library to determine the location of someone on the Internet
8250 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
8252 + This program is free software; you can redistribute it and/or modify
8253 + it under the terms of the GNU General Public License as published by
8254 + the Free Software Foundation; either version 2 of the License, or
8255 + (at your option) any later version.
8257 + This program is distributed in the hope that it will be useful,
8258 + but WITHOUT ANY WARRANTY; without even the implied warranty of
8259 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
8260 + GNU General Public License for more details.
8265 +#include <stddef.h>
8266 +#include <stdlib.h>
8267 +#include <string.h>
8268 +#include <syslog.h>
8270 +#include <loc/libloc.h>
8271 +#include <loc/network.h>
8272 +#include <loc/network-list.h>
8274 +int main(int argc, char** argv) {
8277 + struct loc_ctx* ctx;
8278 + err = loc_new(&ctx);
8280 + exit(EXIT_FAILURE);
8282 + // Enable debug logging
8283 + loc_set_log_priority(ctx, LOG_DEBUG);
8285 + // Create a network
8286 + struct loc_network* network1;
8287 + err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32");
8289 + fprintf(stderr, "Could not create the network1\n");
8290 + exit(EXIT_FAILURE);
8293 + struct loc_network* subnet1;
8294 + err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48");
8296 + fprintf(stderr, "Could not create the subnet1\n");
8297 + exit(EXIT_FAILURE);
8300 + struct loc_network* subnet2;
8301 + err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48");
8303 + fprintf(stderr, "Could not create the subnet2\n");
8304 + exit(EXIT_FAILURE);
8307 + // Make a list with both subnets
8308 + struct loc_network_list* subnets;
8309 + err = loc_network_list_new(ctx, &subnets);
8311 + fprintf(stderr, "Could not create subnets list\n");
8312 + exit(EXIT_FAILURE);
8315 + size_t size = loc_network_list_size(subnets);
8317 + fprintf(stderr, "The list is not empty: %zu\n", size);
8318 + exit(EXIT_FAILURE);
8321 + err = loc_network_list_push(subnets, subnet1);
8323 + fprintf(stderr, "Could not add subnet1 to subnets list\n");
8324 + exit(EXIT_FAILURE);
8327 + if (loc_network_list_empty(subnets)) {
8328 + fprintf(stderr, "The subnets list reports that it is empty\n");
8329 + exit(EXIT_FAILURE);
8332 + err = loc_network_list_push(subnets, subnet2);
8334 + fprintf(stderr, "Could not add subnet2 to subnets list\n");
8335 + exit(EXIT_FAILURE);
8338 + size = loc_network_list_size(subnets);
8340 + fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
8341 + exit(EXIT_FAILURE);
8344 + // Exclude subnet1 from network1
8345 + struct loc_network_list* excluded = loc_network_exclude(network1, subnet1);
8347 + fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n");
8348 + exit(EXIT_FAILURE);
8351 + loc_network_list_dump(excluded);
8353 + // Exclude all subnets from network1
8354 + excluded = loc_network_exclude_list(network1, subnets);
8356 + fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n");
8357 + exit(EXIT_FAILURE);
8360 + loc_network_list_dump(excluded);
8363 + loc_network_list_unref(excluded);
8365 + loc_network_list_unref(subnets);
8366 + loc_network_unref(network1);
8367 + loc_network_unref(subnet1);
8368 + loc_network_unref(subnet2);
8371 + return EXIT_SUCCESS;
8376 From 1209ff0cd4be6b2f52a5e82168db8a8ac68e628a Mon Sep 17 00:00:00 2001
8377 From: Michael Tremer <michael.tremer@ipfire.org>
8378 Date: Fri, 20 Nov 2020 16:24:40 +0000
8379 Subject: [PATCH 079/111] network: Fix loc_network_is_subnet()
8381 This function always returned false.
8383 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
8385 src/network.c | 16 +++-------------
8386 1 file changed, 3 insertions(+), 13 deletions(-)
8388 diff --git a/src/network.c b/src/network.c
8389 index 4c8787a..b440d76 100644
8392 @@ -491,12 +491,12 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
8393 LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
8394 // If the start address of the other network is smaller than this network,
8395 // it cannot be a subnet.
8396 - if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
8397 + if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
8400 // If the end address of the other network is greater than this network,
8401 // it cannot be a subnet.
8402 - if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
8403 + if (in6_addr_cmp(&self->last_address, &other->last_address) < 0)
8407 @@ -504,17 +504,7 @@ LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_networ
8409 // XXX DEPRECATED - I find this too difficult to use
8410 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
8411 - // If the start address of the other network is smaller than this network,
8412 - // it cannot be a subnet.
8413 - if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
8416 - // If the end address of the other network is greater than this network,
8417 - // it cannot be a subnet.
8418 - if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
8422 + return loc_network_is_subnet(other, self);
8425 LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
8429 From 92349c14876f8e4711739c82d1a389a53bcb27a5 Mon Sep 17 00:00:00 2001
8430 From: Michael Tremer <michael.tremer@ipfire.org>
8431 Date: Fri, 20 Nov 2020 16:25:27 +0000
8432 Subject: [PATCH 080/111] network: Remove debugging output
8434 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
8436 src/network.c | 4 ----
8437 1 file changed, 4 deletions(-)
8439 diff --git a/src/network.c b/src/network.c
8440 index b440d76..3271272 100644
8443 @@ -670,10 +670,6 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude(
8444 loc_network_unref(subnet2);
8447 -#ifdef ENABLE_DEBUG
8448 - loc_network_list_dump(list);
8451 // Return the result
8457 From ed0f53df0f3ebb915faf25138cc09df7555415a3 Mon Sep 17 00:00:00 2001
8458 From: Michael Tremer <michael.tremer@ipfire.org>
8459 Date: Fri, 20 Nov 2020 16:25:56 +0000
8460 Subject: [PATCH 081/111] database: Flatten out code due to compiler errors
8462 It looks like GCC could not follow the code as it was
8463 written. Therefore I clean up more often now instead
8464 of having a GOTO block where I do that in.
8466 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
8468 src/database.c | 59 ++++++++++++++++++++++++++++++++++----------------
8469 1 file changed, 40 insertions(+), 19 deletions(-)
8471 diff --git a/src/database.c b/src/database.c
8472 index ef4f505..c21b957 100644
8473 --- a/src/database.c
8474 +++ b/src/database.c
8475 @@ -1305,8 +1305,12 @@ static int __loc_database_enumerator_next_network_flattened(
8477 // Fetch the next network in line
8478 r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
8482 + loc_network_unref(subnet);
8483 + loc_network_list_unref(subnets);
8488 // End if we did not receive another subnet
8490 @@ -1315,8 +1319,12 @@ static int __loc_database_enumerator_next_network_flattened(
8491 // Collect all subnets in a list
8492 if (loc_network_is_subnet(*network, subnet)) {
8493 r = loc_network_list_push(subnets, subnet);
8497 + loc_network_unref(subnet);
8498 + loc_network_list_unref(subnets);
8503 loc_network_unref(subnet);
8505 @@ -1324,8 +1332,12 @@ static int __loc_database_enumerator_next_network_flattened(
8507 // If this is not a subnet, we push it back onto the stack and break
8508 r = loc_network_list_push(enumerator->stack, subnet);
8512 + loc_network_unref(subnet);
8513 + loc_network_list_unref(subnets);
8518 loc_network_unref(subnet);
8520 @@ -1343,29 +1355,38 @@ static int __loc_database_enumerator_next_network_flattened(
8521 // If the network has any subnets, we will break it into smaller parts
8522 // without the subnets.
8523 struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
8524 - if (!excluded || loc_network_list_empty(excluded)) {
8528 + loc_network_list_unref(subnets);
8532 + // Merge excluded list with subnets
8533 + r = loc_network_list_merge(subnets, excluded);
8535 + loc_network_list_unref(subnets);
8536 + loc_network_list_unref(excluded);
8541 + // We no longer need the excluded list
8542 + loc_network_list_unref(excluded);
8544 + // Sort all subnets
8545 + loc_network_list_sort(subnets);
8547 // Replace network with the first one
8548 loc_network_unref(*network);
8550 - *network = loc_network_list_pop_first(excluded);
8551 + *network = loc_network_list_pop_first(subnets);
8553 // Push the rest onto the stack
8554 - loc_network_list_reverse(excluded);
8555 - loc_network_list_merge(enumerator->stack, excluded);
8557 - loc_network_list_unref(excluded);
8561 - loc_network_unref(subnet);
8562 + loc_network_list_reverse(subnets);
8563 + loc_network_list_merge(enumerator->stack, subnets);
8565 loc_network_list_unref(subnets);
8571 LOC_EXPORT int loc_database_enumerator_next_network(
8575 From a1024390795d60fab9f697fa230c9901ebd7d221 Mon Sep 17 00:00:00 2001
8576 From: Michael Tremer <michael.tremer@ipfire.org>
8577 Date: Fri, 20 Nov 2020 16:42:55 +0000
8578 Subject: [PATCH 082/111] network-list: Implement merging in reverse in one
8581 This will save us some time because we do not need to
8582 change the list in place first and then merge it.
8584 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
8586 src/database.c | 3 +--
8587 src/libloc.sym | 1 +
8588 src/loc/network-list.h | 1 +
8589 src/network-list.c | 13 +++++++++++++
8590 4 files changed, 16 insertions(+), 2 deletions(-)
8592 diff --git a/src/database.c b/src/database.c
8593 index c21b957..ba4f98a 100644
8594 --- a/src/database.c
8595 +++ b/src/database.c
8596 @@ -1381,8 +1381,7 @@ static int __loc_database_enumerator_next_network_flattened(
8597 *network = loc_network_list_pop_first(subnets);
8599 // Push the rest onto the stack
8600 - loc_network_list_reverse(subnets);
8601 - loc_network_list_merge(enumerator->stack, subnets);
8602 + loc_network_list_merge_reverse(enumerator->stack, subnets);
8604 loc_network_list_unref(subnets);
8606 diff --git a/src/libloc.sym b/src/libloc.sym
8607 index 406dd15..56cada8 100644
8608 --- a/src/libloc.sym
8609 +++ b/src/libloc.sym
8610 @@ -140,6 +140,7 @@ global:
8611 loc_network_list_empty;
8612 loc_network_list_get;
8613 loc_network_list_merge;
8614 + loc_network_list_merge_reverse;
8615 loc_network_list_new;
8616 loc_network_list_pop;
8617 loc_network_list_pop_first;
8618 diff --git a/src/loc/network-list.h b/src/loc/network-list.h
8619 index af3b28d..89776a6 100644
8620 --- a/src/loc/network-list.h
8621 +++ b/src/loc/network-list.h
8622 @@ -33,5 +33,6 @@ int loc_network_list_contains(struct loc_network_list* list, struct loc_network*
8623 void loc_network_list_sort(struct loc_network_list* list);
8624 void loc_network_list_reverse(struct loc_network_list* list);
8625 int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
8626 +int loc_network_list_merge_reverse(struct loc_network_list* self, struct loc_network_list* other);
8629 diff --git a/src/network-list.c b/src/network-list.c
8630 index 9cb4547..b455caf 100644
8631 --- a/src/network-list.c
8632 +++ b/src/network-list.c
8633 @@ -242,3 +242,16 @@ LOC_EXPORT int loc_network_list_merge(
8638 +LOC_EXPORT int loc_network_list_merge_reverse(
8639 + struct loc_network_list* self, struct loc_network_list* other) {
8642 + for (int i = other->size - 1; i >= 0; i--) {
8643 + r = loc_network_list_push(self, other->elements[i]);
8653 From a5967330d530504db401540d4bcd5474fe00e421 Mon Sep 17 00:00:00 2001
8654 From: Michael Tremer <michael.tremer@ipfire.org>
8655 Date: Fri, 20 Nov 2020 18:36:48 +0000
8656 Subject: [PATCH 083/111] network: Optimise _subnet function
8658 This function used to create a network list which always
8659 had exactly two elements. Since splitting a network in half
8660 always returns two parts, we can simply return them as a
8663 This improves returning the network tree by about 17%.
8665 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
8667 src/loc/network.h | 2 +-
8668 src/network.c | 158 ++++++++++++++++++---------------------------
8669 src/test-network.c | 51 ++++++++-------
8670 3 files changed, 91 insertions(+), 120 deletions(-)
8672 diff --git a/src/loc/network.h b/src/loc/network.h
8673 index 4b7410c..d67ec54 100644
8674 --- a/src/loc/network.h
8675 +++ b/src/loc/network.h
8676 @@ -62,7 +62,7 @@ int loc_network_gt(struct loc_network* self, struct loc_network* other);
8677 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
8678 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
8679 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
8680 -struct loc_network_list* loc_network_subnets(struct loc_network* network);
8681 +int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
8682 struct loc_network_list* loc_network_exclude(
8683 struct loc_network* self, struct loc_network* other);
8684 struct loc_network_list* loc_network_exclude_list(
8685 diff --git a/src/network.c b/src/network.c
8686 index 3271272..ff9c1eb 100644
8689 @@ -507,66 +507,91 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net
8690 return loc_network_is_subnet(other, self);
8693 -LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
8694 - struct loc_network_list* list;
8695 +LOC_EXPORT int loc_network_subnets(struct loc_network* network,
8696 + struct loc_network** subnet1, struct loc_network** subnet2) {
8701 // New prefix length
8702 unsigned int prefix = network->prefix + 1;
8704 // Check if the new prefix is valid
8705 if (valid_prefix(&network->first_address, prefix))
8708 - // Create a new list with the result
8709 - int r = loc_network_list_new(network->ctx, &list);
8711 - ERROR(network->ctx, "Could not create network list: %d\n", r);
8715 - struct loc_network* subnet1 = NULL;
8716 - struct loc_network* subnet2 = NULL;
8719 // Create the first half of the network
8720 - r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix);
8721 + r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix);
8726 // The next subnet starts after the first one
8727 - struct in6_addr first_address = address_increment(&subnet1->last_address);
8728 + struct in6_addr first_address = address_increment(&(*subnet1)->last_address);
8730 // Create the second half of the network
8731 - r = loc_network_new(network->ctx, &subnet2, &first_address, prefix);
8735 - // Push the both onto the stack (in reverse order)
8736 - r = loc_network_list_push(list, subnet2);
8737 + r = loc_network_new(network->ctx, subnet2, &first_address, prefix);
8741 - r = loc_network_list_push(list, subnet1);
8746 // Copy country code
8747 const char* country_code = loc_network_get_country_code(network);
8749 - loc_network_set_country_code(subnet1, country_code);
8750 - loc_network_set_country_code(subnet2, country_code);
8751 + loc_network_set_country_code(*subnet1, country_code);
8752 + loc_network_set_country_code(*subnet2, country_code);
8756 uint32_t asn = loc_network_get_asn(network);
8758 - loc_network_set_asn(subnet1, asn);
8759 - loc_network_set_asn(subnet2, asn);
8760 + loc_network_set_asn(*subnet1, asn);
8761 + loc_network_set_asn(*subnet2, asn);
8764 - loc_network_unref(subnet1);
8765 - loc_network_unref(subnet2);
8770 +static int __loc_network_exclude(struct loc_network* network,
8771 + struct loc_network* other, struct loc_network_list* list) {
8772 + struct loc_network* subnet1 = NULL;
8773 + struct loc_network* subnet2 = NULL;
8775 + int r = loc_network_subnets(network, &subnet1, &subnet2);
8779 + if (loc_network_eq(other, subnet1)) {
8780 + r = loc_network_list_push(list, subnet2);
8784 + } else if (loc_network_eq(other, subnet2)) {
8785 + r = loc_network_list_push(list, subnet1);
8789 + } else if (loc_network_is_subnet_of(other, subnet1)) {
8790 + r = loc_network_list_push(list, subnet2);
8794 + r = __loc_network_exclude(subnet1, other, list);
8798 + } else if (loc_network_is_subnet_of(other, subnet2)) {
8799 + r = loc_network_list_push(list, subnet1);
8803 + r = __loc_network_exclude(subnet2, other, list);
8808 + ERROR(network->ctx, "We should never get here\n");
8815 @@ -575,10 +600,7 @@ ERROR:
8817 loc_network_unref(subnet2);
8820 - loc_network_list_unref(list);
8826 LOC_EXPORT struct loc_network_list* loc_network_exclude(
8827 @@ -623,67 +645,15 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude(
8831 - struct loc_network_list* subnets = loc_network_subnets(self);
8833 - struct loc_network* subnet1 = NULL;
8834 - struct loc_network* subnet2 = NULL;
8837 - // Fetch both subnets
8838 - subnet1 = loc_network_list_get(subnets, 0);
8839 - subnet2 = loc_network_list_get(subnets, 1);
8842 - loc_network_list_unref(subnets);
8845 - if (loc_network_eq(other, subnet1)) {
8846 - r = loc_network_list_push(list, subnet2);
8850 - } else if (loc_network_eq(other, subnet2)) {
8851 - r = loc_network_list_push(list, subnet1);
8855 - } else if (loc_network_is_subnet_of(other, subnet1)) {
8856 - r = loc_network_list_push(list, subnet2);
8860 - subnets = loc_network_subnets(subnet1);
8862 - } else if (loc_network_is_subnet_of(other, subnet2)) {
8863 - r = loc_network_list_push(list, subnet1);
8867 - subnets = loc_network_subnets(subnet2);
8870 - ERROR(self->ctx, "We should never get here\n");
8873 + r = __loc_network_exclude(self, other, list);
8875 + loc_network_list_unref(list);
8877 - loc_network_unref(subnet1);
8878 - loc_network_unref(subnet2);
8882 // Return the result
8887 - loc_network_unref(subnet1);
8890 - loc_network_unref(subnet2);
8893 - loc_network_list_unref(list);
8898 LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
8899 diff --git a/src/test-network.c b/src/test-network.c
8900 index 7c90224..79c2967 100644
8901 --- a/src/test-network.c
8902 +++ b/src/test-network.c
8903 @@ -138,47 +138,48 @@ int main(int argc, char** argv) {
8906 // Check subnet function
8907 - err = loc_network_is_subnet_of(network1, network2);
8909 + err = loc_network_is_subnet(network1, network2);
8911 fprintf(stderr, "Subnet check 1 failed: %d\n", err);
8915 - err = loc_network_is_subnet_of(network2, network1);
8917 + err = loc_network_is_subnet(network2, network1);
8919 fprintf(stderr, "Subnet check 2 failed: %d\n", err);
8923 - // Make list of subnets
8924 - struct loc_network_list* subnets = loc_network_subnets(network1);
8926 - fprintf(stderr, "Could not find subnets of network\n");
8928 + struct loc_network* subnet1 = NULL;
8929 + struct loc_network* subnet2 = NULL;
8931 + err = loc_network_subnets(network1, &subnet1, &subnet2);
8932 + if (err || !subnet1 || !subnet2) {
8933 + fprintf(stderr, "Could not find subnets of network: %d\n", err);
8937 - loc_network_list_dump(subnets);
8939 - while (!loc_network_list_empty(subnets)) {
8940 - struct loc_network* subnet = loc_network_list_pop(subnets);
8942 - fprintf(stderr, "Received an empty subnet\n");
8943 - exit(EXIT_FAILURE);
8945 + char* s = loc_network_str(subnet1);
8946 + printf("Received subnet1 = %s\n", s);
8949 - char* s = loc_network_str(subnet);
8950 - printf("Received subnet %s\n", s);
8952 + s = loc_network_str(subnet2);
8953 + printf("Received subnet2 = %s\n", s);
8956 - if (!loc_network_is_subnet_of(subnet, network1)) {
8957 - fprintf(stderr, "Not a subnet\n");
8958 - exit(EXIT_FAILURE);
8960 + if (!loc_network_is_subnet(network1, subnet1)) {
8961 + fprintf(stderr, "Subnet1 is not a subnet\n");
8962 + exit(EXIT_FAILURE);
8965 - loc_network_unref(subnet);
8966 + if (!loc_network_is_subnet(network1, subnet2)) {
8967 + fprintf(stderr, "Subnet2 is not a subnet\n");
8968 + exit(EXIT_FAILURE);
8971 - loc_network_list_unref(subnets);
8972 + loc_network_unref(subnet1);
8973 + loc_network_unref(subnet2);
8975 struct loc_network_list* excluded = loc_network_exclude(network1, network2);
8980 From 7fe6a21845edf6692d239f228117bc95620d0419 Mon Sep 17 00:00:00 2001
8981 From: Michael Tremer <michael.tremer@ipfire.org>
8982 Date: Fri, 20 Nov 2020 18:38:47 +0000
8983 Subject: [PATCH 084/111] network: Add function to return the prefix
8985 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
8987 src/libloc.sym | 1 +
8988 src/loc/network.h | 1 +
8989 src/network.c | 12 ++++++++++++
8990 3 files changed, 14 insertions(+)
8992 diff --git a/src/libloc.sym b/src/libloc.sym
8993 index 56cada8..9f41c89 100644
8994 --- a/src/libloc.sym
8995 +++ b/src/libloc.sym
8996 @@ -125,6 +125,7 @@ global:
8998 loc_network_new_from_string;
8999 loc_network_overlaps;
9000 + loc_network_prefix;
9002 loc_network_set_asn;
9003 loc_network_set_country_code;
9004 diff --git a/src/loc/network.h b/src/loc/network.h
9005 index d67ec54..7b2ae4c 100644
9006 --- a/src/loc/network.h
9007 +++ b/src/loc/network.h
9008 @@ -38,6 +38,7 @@ struct loc_network* loc_network_ref(struct loc_network* network);
9009 struct loc_network* loc_network_unref(struct loc_network* network);
9010 char* loc_network_str(struct loc_network* network);
9011 int loc_network_address_family(struct loc_network* network);
9012 +unsigned int loc_network_prefix(struct loc_network* network);
9014 const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
9015 char* loc_network_format_first_address(struct loc_network* network);
9016 diff --git a/src/network.c b/src/network.c
9017 index ff9c1eb..4720503 100644
9020 @@ -310,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) {
9021 return network->family;
9024 +LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) {
9025 + switch (network->family) {
9027 + return network->prefix;
9030 + return network->prefix - 96;
9036 static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) {
9037 const size_t length = INET6_ADDRSTRLEN;
9042 From cf8d3c6454843943e3bc81eb85522779d1c11f9b Mon Sep 17 00:00:00 2001
9043 From: Michael Tremer <michael.tremer@ipfire.org>
9044 Date: Fri, 20 Nov 2020 18:39:13 +0000
9045 Subject: [PATCH 085/111] network: Speed up subnet check
9047 There is no point in checking different address families
9048 with each other and we do not need to compare addresses
9049 when the prefix of the subnet does not fit into the
9052 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
9054 src/network.c | 8 ++++++++
9055 1 file changed, 8 insertions(+)
9057 diff --git a/src/network.c b/src/network.c
9058 index 4720503..e52b58c 100644
9061 @@ -501,6 +501,14 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
9064 LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
9066 + if (self->family != other->family)
9069 + // The prefix must be smaller (this avoids the more complex comparisons later)
9070 + if (self->prefix > other->prefix)
9073 // If the start address of the other network is smaller than this network,
9074 // it cannot be a subnet.
9075 if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
9079 From d6a5092f969bb3bd50d130d4ba64b4e4be2e61f6 Mon Sep 17 00:00:00 2001
9080 From: Michael Tremer <michael.tremer@ipfire.org>
9081 Date: Fri, 20 Nov 2020 18:44:42 +0000
9082 Subject: [PATCH 086/111] network: Remove deprecated loc_network_is_subnet_of
9085 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
9087 src/libloc.sym | 1 -
9088 src/loc/network.h | 1 -
9089 src/network.c | 13 ++++---------
9090 src/python/network.c | 2 +-
9091 4 files changed, 5 insertions(+), 12 deletions(-)
9093 diff --git a/src/libloc.sym b/src/libloc.sym
9094 index 9f41c89..b2d8a31 100644
9095 --- a/src/libloc.sym
9096 +++ b/src/libloc.sym
9097 @@ -118,7 +118,6 @@ global:
9099 loc_network_has_flag;
9100 loc_network_is_subnet;
9101 - loc_network_is_subnet_of;
9102 loc_network_match_asn;
9103 loc_network_match_country_code;
9104 loc_network_match_flag;
9105 diff --git a/src/loc/network.h b/src/loc/network.h
9106 index 7b2ae4c..b31c8a2 100644
9107 --- a/src/loc/network.h
9108 +++ b/src/loc/network.h
9109 @@ -62,7 +62,6 @@ int loc_network_eq(struct loc_network* self, struct loc_network* other);
9110 int loc_network_gt(struct loc_network* self, struct loc_network* other);
9111 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
9112 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
9113 -int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
9114 int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
9115 struct loc_network_list* loc_network_exclude(
9116 struct loc_network* self, struct loc_network* other);
9117 diff --git a/src/network.c b/src/network.c
9118 index e52b58c..72b77e6 100644
9121 @@ -522,11 +522,6 @@ LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_networ
9125 -// XXX DEPRECATED - I find this too difficult to use
9126 -LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
9127 - return loc_network_is_subnet(other, self);
9130 LOC_EXPORT int loc_network_subnets(struct loc_network* network,
9131 struct loc_network** subnet1, struct loc_network** subnet2) {
9133 @@ -589,7 +584,7 @@ static int __loc_network_exclude(struct loc_network* network,
9137 - } else if (loc_network_is_subnet_of(other, subnet1)) {
9138 + } else if (loc_network_is_subnet(subnet1, other)) {
9139 r = loc_network_list_push(list, subnet2);
9142 @@ -598,7 +593,7 @@ static int __loc_network_exclude(struct loc_network* network,
9146 - } else if (loc_network_is_subnet_of(other, subnet2)) {
9147 + } else if (loc_network_is_subnet(subnet2, other)) {
9148 r = loc_network_list_push(list, subnet1);
9151 @@ -645,7 +640,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude(
9154 // Other must be a subnet of self
9155 - if (!loc_network_is_subnet_of(other, self)) {
9156 + if (!loc_network_is_subnet(self, other)) {
9157 DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
9160 @@ -726,7 +721,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
9163 // Drop this subnet if is a subnet of another subnet
9164 - if (loc_network_is_subnet_of(subnet, subnet_to_check)) {
9165 + if (loc_network_is_subnet(subnet_to_check, subnet)) {
9167 loc_network_unref(subnet);
9169 diff --git a/src/python/network.c b/src/python/network.c
9170 index 742b472..b6e92fb 100644
9171 --- a/src/python/network.c
9172 +++ b/src/python/network.c
9173 @@ -194,7 +194,7 @@ static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
9174 if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
9177 - if (loc_network_is_subnet_of(self->network, other->network))
9178 + if (loc_network_is_subnet(other->network, self->network))
9185 From af4689bf5c56ad79e9e90396b41be460e49ef384 Mon Sep 17 00:00:00 2001
9186 From: Michael Tremer <michael.tremer@ipfire.org>
9187 Date: Tue, 24 Nov 2020 14:55:08 +0000
9188 Subject: [PATCH 087/111] network-list: Make this a sorted list
9190 The list is now organised so that it is always ordered.
9192 This allows us to find if a network is on the list a lot
9193 quicker using binary search as well as inserting a new
9194 network at the right place.
9196 This will benefit loc_network_exclude with very large networks.
9198 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
9200 src/libloc.sym | 1 +
9201 src/loc/network.h | 1 +
9202 src/network-list.c | 84 +++++++++++++++++++++++++++++++++++++++++-----
9203 src/network.c | 22 ++++++++++++
9204 4 files changed, 99 insertions(+), 9 deletions(-)
9206 diff --git a/src/libloc.sym b/src/libloc.sym
9207 index b2d8a31..28cc8e8 100644
9208 --- a/src/libloc.sym
9209 +++ b/src/libloc.sym
9210 @@ -106,6 +106,7 @@ global:
9213 loc_network_address_family;
9216 loc_network_exclude;
9217 loc_network_exclude_list;
9218 diff --git a/src/loc/network.h b/src/loc/network.h
9219 index b31c8a2..8ab1562 100644
9220 --- a/src/loc/network.h
9221 +++ b/src/loc/network.h
9222 @@ -58,6 +58,7 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
9223 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
9224 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
9226 +int loc_network_cmp(struct loc_network* self, struct loc_network* other);
9227 int loc_network_eq(struct loc_network* self, struct loc_network* other);
9228 int loc_network_gt(struct loc_network* self, struct loc_network* other);
9229 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
9230 diff --git a/src/network-list.c b/src/network-list.c
9231 index b455caf..6e9cd37 100644
9232 --- a/src/network-list.c
9233 +++ b/src/network-list.c
9240 #include <loc/libloc.h>
9241 #include <loc/network.h>
9242 @@ -130,11 +131,71 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
9243 return loc_network_ref(list->elements[index]);
9246 +//MOVE FUNCTION GOES HERE
9248 +static off_t loc_network_list_find(struct loc_network_list* list,
9249 + struct loc_network* network, int* found) {
9251 + off_t hi = list->size - 1;
9255 +#ifdef ENABLE_DEBUG
9256 + // Save start time
9257 + clock_t start = clock();
9262 + while (lo <= hi) {
9263 + i = (lo + hi) / 2;
9265 + // Check if this is a match
9266 + int result = loc_network_cmp(network, list->elements[i]);
9268 + if (result == 0) {
9271 +#ifdef ENABLE_DEBUG
9272 + clock_t end = clock();
9274 + // Log how fast this has been
9275 + DEBUG(list->ctx, "Found network in %.4fms at %jd\n",
9276 + (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
9288 +#ifdef ENABLE_DEBUG
9289 + clock_t end = clock();
9291 + // Log how fast this has been
9292 + DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n",
9293 + (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
9299 LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
9300 - // Do not add networks that are already on the list
9301 - if (loc_network_list_contains(list, network))
9304 + off_t index = loc_network_list_find(list, network, &found);
9306 + // The network has been found on the list. Nothing to do.
9310 + DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n",
9311 + list, network, (intmax_t)index);
9313 // Check if we have space left
9314 if (list->size >= list->elements_size) {
9315 int r = loc_network_list_grow(list, 64);
9316 @@ -142,9 +203,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
9320 - DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
9321 + // The list is now larger
9324 + // Move all elements out of the way
9325 + for (unsigned int i = list->size - 1; i > index; i--)
9326 + list->elements[i] = list->elements[i - 1];
9328 - list->elements[list->size++] = loc_network_ref(network);
9329 + // Add the new element at the right place
9330 + list->elements[index] = loc_network_ref(network);
9334 @@ -183,12 +250,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
9337 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
9338 - for (unsigned int i = 0; i < list->size; i++) {
9339 - if (loc_network_eq(list->elements[i], network))
9345 + loc_network_list_find(list, network, &found);
9350 static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
9351 diff --git a/src/network.c b/src/network.c
9352 index 72b77e6..38d557a 100644
9355 @@ -441,6 +441,28 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
9356 return loc_network_has_flag(network, flag);
9359 +LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
9361 + if (self->family > other->family)
9363 + else if (self->family < other->family)
9366 + // Compare address
9367 + int r = in6_addr_cmp(&self->first_address, &other->first_address);
9372 + if (self->prefix > other->prefix)
9374 + else if (self->prefix < other->prefix)
9377 + // Both networks are equal
9381 LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
9382 // Family must be the same
9383 if (self->family != other->family)
9387 From 39cbbc63aee362d82f69a9b4722b59153ce799a0 Mon Sep 17 00:00:00 2001
9388 From: Michael Tremer <michael.tremer@ipfire.org>
9389 Date: Tue, 24 Nov 2020 15:04:03 +0000
9390 Subject: [PATCH 088/111] network-list: Drop sorting functions
9392 Since the list is always sorted, there is no point in
9395 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
9397 src/database.c | 6 +----
9398 src/libloc.sym | 3 ---
9399 src/loc/network-list.h | 3 ---
9400 src/network-list.c | 52 ------------------------------------------
9401 src/test-network.c | 9 --------
9402 5 files changed, 1 insertion(+), 72 deletions(-)
9404 diff --git a/src/database.c b/src/database.c
9405 index ba4f98a..5546091 100644
9406 --- a/src/database.c
9407 +++ b/src/database.c
9408 @@ -1372,17 +1372,13 @@ static int __loc_database_enumerator_next_network_flattened(
9409 // We no longer need the excluded list
9410 loc_network_list_unref(excluded);
9412 - // Sort all subnets
9413 - loc_network_list_sort(subnets);
9415 // Replace network with the first one
9416 loc_network_unref(*network);
9418 *network = loc_network_list_pop_first(subnets);
9420 // Push the rest onto the stack
9421 - loc_network_list_merge_reverse(enumerator->stack, subnets);
9423 + loc_network_list_merge(enumerator->stack, subnets);
9424 loc_network_list_unref(subnets);
9427 diff --git a/src/libloc.sym b/src/libloc.sym
9428 index 28cc8e8..4b0ce45 100644
9429 --- a/src/libloc.sym
9430 +++ b/src/libloc.sym
9431 @@ -141,15 +141,12 @@ global:
9432 loc_network_list_empty;
9433 loc_network_list_get;
9434 loc_network_list_merge;
9435 - loc_network_list_merge_reverse;
9436 loc_network_list_new;
9437 loc_network_list_pop;
9438 loc_network_list_pop_first;
9439 loc_network_list_push;
9440 loc_network_list_ref;
9441 - loc_network_list_reverse;
9442 loc_network_list_size;
9443 - loc_network_list_sort;
9444 loc_network_list_unref;
9447 diff --git a/src/loc/network-list.h b/src/loc/network-list.h
9448 index 89776a6..21c7402 100644
9449 --- a/src/loc/network-list.h
9450 +++ b/src/loc/network-list.h
9451 @@ -30,9 +30,6 @@ int loc_network_list_push(struct loc_network_list* list, struct loc_network* net
9452 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
9453 struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
9454 int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
9455 -void loc_network_list_sort(struct loc_network_list* list);
9456 -void loc_network_list_reverse(struct loc_network_list* list);
9457 int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
9458 -int loc_network_list_merge_reverse(struct loc_network_list* self, struct loc_network_list* other);
9461 diff --git a/src/network-list.c b/src/network-list.c
9462 index 6e9cd37..7e8b5f3 100644
9463 --- a/src/network-list.c
9464 +++ b/src/network-list.c
9465 @@ -257,45 +257,6 @@ LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct l
9469 -static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
9470 - // Do nothing for invalid indices
9471 - if (i1 >= list->size || i2 >= list->size)
9474 - struct loc_network* network1 = list->elements[i1];
9475 - struct loc_network* network2 = list->elements[i2];
9477 - list->elements[i1] = network2;
9478 - list->elements[i2] = network1;
9481 -LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
9482 - unsigned int i = 0;
9483 - unsigned int j = list->size - 1;
9486 - loc_network_list_swap(list, i++, j--);
9490 -LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
9491 - unsigned int n = list->size;
9497 - for (unsigned int i = 1; i < n; i++) {
9498 - if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) {
9499 - loc_network_list_swap(list, i-1, i);
9505 - } while (swapped);
9508 LOC_EXPORT int loc_network_list_merge(
9509 struct loc_network_list* self, struct loc_network_list* other) {
9511 @@ -308,16 +269,3 @@ LOC_EXPORT int loc_network_list_merge(
9516 -LOC_EXPORT int loc_network_list_merge_reverse(
9517 - struct loc_network_list* self, struct loc_network_list* other) {
9520 - for (int i = other->size - 1; i >= 0; i--) {
9521 - r = loc_network_list_push(self, other->elements[i]);
9528 diff --git a/src/test-network.c b/src/test-network.c
9529 index 79c2967..8a6763c 100644
9530 --- a/src/test-network.c
9531 +++ b/src/test-network.c
9532 @@ -188,15 +188,6 @@ int main(int argc, char** argv) {
9535 loc_network_list_dump(excluded);
9538 - loc_network_list_reverse(excluded);
9539 - loc_network_list_dump(excluded);
9541 - // Sort them and dump them again
9542 - loc_network_list_sort(excluded);
9543 - loc_network_list_dump(excluded);
9545 loc_network_list_unref(excluded);
9547 // Create a database
9551 From 5dacb45afceac2d05ea597755c1ca5a1b62cc0fd Mon Sep 17 00:00:00 2001
9552 From: Michael Tremer <michael.tremer@ipfire.org>
9553 Date: Tue, 24 Nov 2020 15:15:59 +0000
9554 Subject: [PATCH 089/111] database: Avoid merging the same data twice
9556 When finish splitting networks into many parts, we have
9557 a list of subnets with the excluded subnets and merge them
9558 together first and put them on the stack again.
9560 This is slower than pushing it all onto the stack first
9561 and then popping the first element.
9563 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
9565 src/database.c | 24 ++++++++++++++----------
9566 1 file changed, 14 insertions(+), 10 deletions(-)
9568 diff --git a/src/database.c b/src/database.c
9569 index 5546091..f1f6ae0 100644
9570 --- a/src/database.c
9571 +++ b/src/database.c
9572 @@ -1360,8 +1360,8 @@ static int __loc_database_enumerator_next_network_flattened(
9576 - // Merge excluded list with subnets
9577 - r = loc_network_list_merge(subnets, excluded);
9578 + // Merge subnets onto the stack
9579 + r = loc_network_list_merge(enumerator->stack, subnets);
9581 loc_network_list_unref(subnets);
9582 loc_network_list_unref(excluded);
9583 @@ -1369,17 +1369,21 @@ static int __loc_database_enumerator_next_network_flattened(
9587 - // We no longer need the excluded list
9588 - loc_network_list_unref(excluded);
9590 - // Replace network with the first one
9591 - loc_network_unref(*network);
9592 + // Push excluded list onto the stack
9593 + r = loc_network_list_merge(enumerator->stack, excluded);
9595 + loc_network_list_unref(subnets);
9596 + loc_network_list_unref(excluded);
9598 - *network = loc_network_list_pop_first(subnets);
9602 - // Push the rest onto the stack
9603 - loc_network_list_merge(enumerator->stack, subnets);
9604 loc_network_list_unref(subnets);
9605 + loc_network_list_unref(excluded);
9607 + // Replace network with the first one from the stack
9608 + loc_network_unref(*network);
9609 + *network = loc_network_list_pop_first(enumerator->stack);
9616 From 04cbd2bfa892fc7374ad506ec7ba81727c5a4b96 Mon Sep 17 00:00:00 2001
9617 From: Michael Tremer <michael.tremer@ipfire.org>
9618 Date: Tue, 24 Nov 2020 15:22:02 +0000
9619 Subject: [PATCH 090/111] database: Read the first element from the list
9621 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
9623 src/database.c | 2 +-
9624 1 file changed, 1 insertion(+), 1 deletion(-)
9626 diff --git a/src/database.c b/src/database.c
9627 index f1f6ae0..914ed3e 100644
9628 --- a/src/database.c
9629 +++ b/src/database.c
9630 @@ -1191,7 +1191,7 @@ static int __loc_database_enumerator_next_network(
9631 struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
9632 // Return top element from the stack
9634 - *network = loc_network_list_pop(enumerator->stack);
9635 + *network = loc_network_list_pop_first(enumerator->stack);
9642 From c650008e2dd9af5fd1eadba6354aa0b615047f84 Mon Sep 17 00:00:00 2001
9643 From: Michael Tremer <michael.tremer@ipfire.org>
9644 Date: Tue, 24 Nov 2020 15:41:53 +0000
9645 Subject: [PATCH 091/111] network: Add excluded networks to to_check list
9647 This is now being done immediately instead of creating a new
9648 list which is being merged later.
9650 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
9652 src/network.c | 55 ++++++++++++++++++++++++++++++---------------------
9653 1 file changed, 32 insertions(+), 23 deletions(-)
9655 diff --git a/src/network.c b/src/network.c
9656 index 38d557a..7ab22f8 100644
9659 @@ -640,49 +640,55 @@ ERROR:
9663 -LOC_EXPORT struct loc_network_list* loc_network_exclude(
9664 - struct loc_network* self, struct loc_network* other) {
9665 - struct loc_network_list* list;
9667 -#ifdef ENABLE_DEBUG
9668 - char* n1 = loc_network_str(self);
9669 - char* n2 = loc_network_str(other);
9671 - DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
9677 +static int __loc_network_exclude_to_list(struct loc_network* self,
9678 + struct loc_network* other, struct loc_network_list* list) {
9679 // Family must match
9680 if (self->family != other->family) {
9681 DEBUG(self->ctx, "Family mismatch\n");
9687 // Other must be a subnet of self
9688 if (!loc_network_is_subnet(self, other)) {
9689 DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
9695 // We cannot perform this operation if both networks equal
9696 if (loc_network_eq(self, other)) {
9697 DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
9703 + return __loc_network_exclude(self, other, list);
9706 +LOC_EXPORT struct loc_network_list* loc_network_exclude(
9707 + struct loc_network* self, struct loc_network* other) {
9708 + struct loc_network_list* list;
9710 +#ifdef ENABLE_DEBUG
9711 + char* n1 = loc_network_str(self);
9712 + char* n2 = loc_network_str(other);
9714 + DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
9720 // Create a new list with the result
9721 int r = loc_network_list_new(self->ctx, &list);
9723 ERROR(self->ctx, "Could not create network list: %d\n", r);
9728 - r = __loc_network_exclude(self, other, list);
9729 + r = __loc_network_exclude_to_list(self, other, list);
9731 loc_network_list_unref(list);
9733 @@ -709,11 +715,14 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
9734 subnet = loc_network_list_get(list, i);
9736 // Find all excluded networks
9737 - struct loc_network_list* excluded = loc_network_exclude(network, subnet);
9739 - // Add them all to the "to check" list
9740 - loc_network_list_merge(to_check, excluded);
9741 - loc_network_list_unref(excluded);
9742 + if (!loc_network_list_contains(to_check, subnet)) {
9743 + r = __loc_network_exclude_to_list(network, subnet, to_check);
9745 + loc_network_list_unref(to_check);
9746 + loc_network_unref(subnet);
9756 From 7c6983ad52724d395446bdbd24d36b2ce22aecfd Mon Sep 17 00:00:00 2001
9757 From: Michael Tremer <michael.tremer@ipfire.org>
9758 Date: Tue, 24 Nov 2020 15:42:54 +0000
9759 Subject: [PATCH 092/111] test: Add more networks to list to see the algorithm
9762 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
9764 src/test-network-list.c | 32 +++++++++++++++++++++++++++++++-
9765 1 file changed, 31 insertions(+), 1 deletion(-)
9767 diff --git a/src/test-network-list.c b/src/test-network-list.c
9768 index 3061d63..8253fc7 100644
9769 --- a/src/test-network-list.c
9770 +++ b/src/test-network-list.c
9771 @@ -58,6 +58,20 @@ int main(int argc, char** argv) {
9775 + struct loc_network* subnet3;
9776 + err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48");
9778 + fprintf(stderr, "Could not create the subnet3\n");
9779 + exit(EXIT_FAILURE);
9782 + struct loc_network* subnet4;
9783 + err = loc_network_new_from_string(ctx, &subnet4, "2001:db8:d::/48");
9785 + fprintf(stderr, "Could not create the subnet4\n");
9786 + exit(EXIT_FAILURE);
9789 // Make a list with both subnets
9790 struct loc_network_list* subnets;
9791 err = loc_network_list_new(ctx, &subnets);
9792 @@ -89,8 +103,24 @@ int main(int argc, char** argv) {
9796 + // Add the fourth one next
9797 + err = loc_network_list_push(subnets, subnet4);
9799 + fprintf(stderr, "Could not add subnet4 to subnets list\n");
9800 + exit(EXIT_FAILURE);
9803 + // Add the third one
9804 + err = loc_network_list_push(subnets, subnet3);
9806 + fprintf(stderr, "Could not add subnet3 to subnets list\n");
9807 + exit(EXIT_FAILURE);
9810 + loc_network_list_dump(subnets);
9812 size = loc_network_list_size(subnets);
9815 fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
9821 From da101d55e66ebaeeed8b0b16829a2022a1af0678 Mon Sep 17 00:00:00 2001
9822 From: Michael Tremer <michael.tremer@ipfire.org>
9823 Date: Tue, 24 Nov 2020 15:48:55 +0000
9824 Subject: [PATCH 093/111] Drop loc_network_eq in favour of loc_network_cmp
9826 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
9828 src/libloc.sym | 1 -
9829 src/loc/network.h | 1 -
9830 src/network.c | 24 ++++--------------------
9831 src/test-network.c | 8 ++++----
9832 4 files changed, 8 insertions(+), 26 deletions(-)
9834 diff --git a/src/libloc.sym b/src/libloc.sym
9835 index 4b0ce45..d8e8f14 100644
9836 --- a/src/libloc.sym
9837 +++ b/src/libloc.sym
9838 @@ -107,7 +107,6 @@ global:
9840 loc_network_address_family;
9843 loc_network_exclude;
9844 loc_network_exclude_list;
9845 loc_network_format_first_address;
9846 diff --git a/src/loc/network.h b/src/loc/network.h
9847 index 8ab1562..d5d0ccd 100644
9848 --- a/src/loc/network.h
9849 +++ b/src/loc/network.h
9850 @@ -59,7 +59,6 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
9851 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
9853 int loc_network_cmp(struct loc_network* self, struct loc_network* other);
9854 -int loc_network_eq(struct loc_network* self, struct loc_network* other);
9855 int loc_network_gt(struct loc_network* self, struct loc_network* other);
9856 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
9857 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
9858 diff --git a/src/network.c b/src/network.c
9859 index 7ab22f8..503bf3d 100644
9862 @@ -463,22 +463,6 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth
9866 -LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
9867 - // Family must be the same
9868 - if (self->family != other->family)
9871 - // The start address must be the same
9872 - if (in6_addr_cmp(&self->first_address, &other->first_address) != 0)
9875 - // The prefix length must be the same
9876 - if (self->prefix != other->prefix)
9882 LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
9883 // Families must match
9884 if (self->family != other->family)
9885 @@ -596,12 +580,12 @@ static int __loc_network_exclude(struct loc_network* network,
9889 - if (loc_network_eq(other, subnet1)) {
9890 + if (loc_network_cmp(other, subnet1) == 0) {
9891 r = loc_network_list_push(list, subnet2);
9895 - } else if (loc_network_eq(other, subnet2)) {
9896 + } else if (loc_network_cmp(other, subnet2) == 0) {
9897 r = loc_network_list_push(list, subnet1);
9900 @@ -657,7 +641,7 @@ static int __loc_network_exclude_to_list(struct loc_network* self,
9903 // We cannot perform this operation if both networks equal
9904 - if (loc_network_eq(self, other)) {
9905 + if (loc_network_cmp(self, other) == 0) {
9906 DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
9909 @@ -745,7 +729,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
9910 subnet = loc_network_list_get(list, i);
9912 // Drop this subnet if is is already in list
9913 - if (loc_network_eq(subnet_to_check, subnet)) {
9914 + if (loc_network_cmp(subnet_to_check, subnet) == 0) {
9916 loc_network_unref(subnet);
9918 diff --git a/src/test-network.c b/src/test-network.c
9919 index 8a6763c..f4cf97b 100644
9920 --- a/src/test-network.c
9921 +++ b/src/test-network.c
9922 @@ -125,14 +125,14 @@ int main(int argc, char** argv) {
9925 // Check equals function
9926 - err = loc_network_eq(network1, network1);
9928 + err = loc_network_cmp(network1, network1);
9930 fprintf(stderr, "Network is not equal with itself\n");
9934 - err = loc_network_eq(network1, network2);
9936 + err = loc_network_cmp(network1, network2);
9938 fprintf(stderr, "Networks equal unexpectedly\n");
9944 From 61a6f6e4bf4493236d796e94f3d721bcf5ba68aa Mon Sep 17 00:00:00 2001
9945 From: Michael Tremer <michael.tremer@ipfire.org>
9946 Date: Tue, 24 Nov 2020 15:50:39 +0000
9947 Subject: [PATCH 094/111] Drop loc_network_gt which is now unused
9949 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
9951 src/libloc.sym | 1 -
9952 src/loc/network.h | 1 -
9953 src/network.c | 27 ---------------------------
9954 3 files changed, 29 deletions(-)
9956 diff --git a/src/libloc.sym b/src/libloc.sym
9957 index d8e8f14..cb5e8ef 100644
9958 --- a/src/libloc.sym
9959 +++ b/src/libloc.sym
9960 @@ -115,7 +115,6 @@ global:
9961 loc_network_get_country_code;
9962 loc_network_get_first_address;
9963 loc_network_get_last_address;
9965 loc_network_has_flag;
9966 loc_network_is_subnet;
9967 loc_network_match_asn;
9968 diff --git a/src/loc/network.h b/src/loc/network.h
9969 index d5d0ccd..af3dafd 100644
9970 --- a/src/loc/network.h
9971 +++ b/src/loc/network.h
9972 @@ -59,7 +59,6 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
9973 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
9975 int loc_network_cmp(struct loc_network* self, struct loc_network* other);
9976 -int loc_network_gt(struct loc_network* self, struct loc_network* other);
9977 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
9978 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
9979 int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
9980 diff --git a/src/network.c b/src/network.c
9981 index 503bf3d..ac478d5 100644
9984 @@ -463,33 +463,6 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth
9988 -LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
9989 - // Families must match
9990 - if (self->family != other->family)
9993 - int r = in6_addr_cmp(&self->first_address, &other->first_address);
10008 - if (self->prefix > other->prefix)
10015 LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
10016 if (loc_network_match_address(self, &other->first_address) == 0)
10021 From fc692a58d9f4ca958f88cfa202250c572a0af6ea Mon Sep 17 00:00:00 2001
10022 From: Michael Tremer <michael.tremer@ipfire.org>
10023 Date: Tue, 24 Nov 2020 16:50:17 +0000
10024 Subject: [PATCH 095/111] network: Adjust return codes of
10025 loc_network_match_address and add test
10027 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10029 src/database.c | 3 +--
10030 src/libloc.sym | 1 +
10031 src/network.c | 16 +++++++++-------
10032 src/test-network.c | 14 ++++++++++++++
10033 4 files changed, 25 insertions(+), 9 deletions(-)
10035 diff --git a/src/database.c b/src/database.c
10036 index 914ed3e..1871b74 100644
10037 --- a/src/database.c
10038 +++ b/src/database.c
10039 @@ -776,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru
10042 // Check if the given IP address is inside the network
10043 - r = loc_network_match_address(*network, address);
10045 + if (!loc_network_match_address(*network, address)) {
10046 DEBUG(db->ctx, "Searched address is not part of the network\n");
10048 loc_network_unref(*network);
10049 diff --git a/src/libloc.sym b/src/libloc.sym
10050 index cb5e8ef..ee333f1 100644
10051 --- a/src/libloc.sym
10052 +++ b/src/libloc.sym
10053 @@ -117,6 +117,7 @@ global:
10054 loc_network_get_last_address;
10055 loc_network_has_flag;
10056 loc_network_is_subnet;
10057 + loc_network_match_address;
10058 loc_network_match_asn;
10059 loc_network_match_country_code;
10060 loc_network_match_flag;
10061 diff --git a/src/network.c b/src/network.c
10062 index ac478d5..febab95 100644
10063 --- a/src/network.c
10064 +++ b/src/network.c
10065 @@ -374,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
10066 LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) {
10067 // Address must be larger than the start address
10068 if (in6_addr_cmp(&network->first_address, address) > 0)
10072 // Address must be smaller than the last address
10073 if (in6_addr_cmp(&network->last_address, address) < 0)
10077 // The address is inside this network
10082 LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) {
10083 @@ -464,16 +464,18 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth
10086 LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
10087 - if (loc_network_match_address(self, &other->first_address) == 0)
10088 + // Either of the start addresses must be in the other subnet
10089 + if (loc_network_match_address(self, &other->first_address))
10092 - if (loc_network_match_address(self, &other->last_address) == 0)
10093 + if (loc_network_match_address(other, &self->first_address))
10096 - if (loc_network_match_address(other, &self->first_address) == 0)
10097 + // Or either of the end addresses is in the other subnet
10098 + if (loc_network_match_address(self, &other->last_address))
10101 - if (loc_network_match_address(other, &self->last_address) == 0)
10102 + if (loc_network_match_address(other, &self->last_address))
10106 diff --git a/src/test-network.c b/src/test-network.c
10107 index f4cf97b..339743d 100644
10108 --- a/src/test-network.c
10109 +++ b/src/test-network.c
10111 GNU General Public License for more details.
10114 +#include <arpa/inet.h>
10117 #include <stddef.h>
10118 @@ -46,6 +47,13 @@ int main(int argc, char** argv) {
10122 + struct in6_addr address;
10123 + err = inet_pton(AF_INET6, "2001:db8::1", &address);
10125 + fprintf(stderr, "Could not parse IP address\n");
10126 + exit(EXIT_FAILURE);
10129 // Create a network
10130 struct loc_network* network1;
10131 err = loc_network_new_from_string(ctx, &network1, "2001:db8::1/32");
10132 @@ -92,6 +100,12 @@ int main(int argc, char** argv) {
10133 exit(EXIT_FAILURE);
10136 + err = loc_network_match_address(network1, &address);
10138 + fprintf(stderr, "Network1 does not match address\n");
10139 + exit(EXIT_FAILURE);
10142 struct loc_network* network2;
10143 err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48");
10148 From 06177d8c6004bf8b54322d92926152a7656f9c2a Mon Sep 17 00:00:00 2001
10149 From: Michael Tremer <michael.tremer@ipfire.org>
10150 Date: Tue, 24 Nov 2020 16:57:28 +0000
10151 Subject: [PATCH 096/111] network-list: Use binary search to find if a network
10154 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10156 src/network.c | 13 ++++++-------
10157 1 file changed, 6 insertions(+), 7 deletions(-)
10159 diff --git a/src/network.c b/src/network.c
10160 index febab95..394bafc 100644
10161 --- a/src/network.c
10162 +++ b/src/network.c
10163 @@ -697,19 +697,18 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
10164 while (!loc_network_list_empty(to_check)) {
10165 struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
10167 + // Check whether the subnet to check is part of the input list
10168 + if (loc_network_list_contains(list, subnet_to_check)) {
10169 + loc_network_unref(subnet_to_check);
10173 // Marks whether this subnet passed all checks
10176 for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
10177 subnet = loc_network_list_get(list, i);
10179 - // Drop this subnet if is is already in list
10180 - if (loc_network_cmp(subnet_to_check, subnet) == 0) {
10182 - loc_network_unref(subnet);
10186 // Drop this subnet if is a subnet of another subnet
10187 if (loc_network_is_subnet(subnet_to_check, subnet)) {
10192 From 77e6d5379ea0de3264be5b8918d620d16e6bcbd3 Mon Sep 17 00:00:00 2001
10193 From: Michael Tremer <michael.tremer@ipfire.org>
10194 Date: Tue, 24 Nov 2020 19:39:35 +0000
10195 Subject: [PATCH 097/111] network-list: Check last element before doing binary
10198 This is helpful because very often we walk through a list in
10199 order and are most interested in the last element.
10201 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10203 src/network-list.c | 24 ++++++++++++++++++++++--
10204 1 file changed, 22 insertions(+), 2 deletions(-)
10206 diff --git a/src/network-list.c b/src/network-list.c
10207 index 7e8b5f3..c86ad07 100644
10208 --- a/src/network-list.c
10209 +++ b/src/network-list.c
10210 @@ -137,8 +137,26 @@ static off_t loc_network_list_find(struct loc_network_list* list,
10211 struct loc_network* network, int* found) {
10213 off_t hi = list->size - 1;
10217 + // Since we are working on an ordered list, there is often a good chance that
10218 + // the network we are looking for is at the end or has to go to the end.
10220 + result = loc_network_cmp(network, list->elements[hi]);
10222 + // Match, so we are done
10223 + if (result == 0) {
10228 + // This needs to be added after the last one
10229 + } else if (result > 0) {
10236 #ifdef ENABLE_DEBUG
10238 @@ -151,7 +169,7 @@ static off_t loc_network_list_find(struct loc_network_list* list,
10241 // Check if this is a match
10242 - int result = loc_network_cmp(network, list->elements[i]);
10243 + result = loc_network_cmp(network, list->elements[i]);
10247 @@ -173,6 +191,8 @@ static off_t loc_network_list_find(struct loc_network_list* list,
10253 #ifdef ENABLE_DEBUG
10254 clock_t end = clock();
10259 From d42f34ce9b0aa2382ae9b852882c62b34d367cfe Mon Sep 17 00:00:00 2001
10260 From: Michael Tremer <michael.tremer@ipfire.org>
10261 Date: Wed, 25 Nov 2020 14:41:39 +0000
10262 Subject: [PATCH 098/111] network-list: Set elements pointer to NULL so that we
10265 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10267 src/network-list.c | 1 +
10268 1 file changed, 1 insertion(+)
10270 diff --git a/src/network-list.c b/src/network-list.c
10271 index c86ad07..2c4edb3 100644
10272 --- a/src/network-list.c
10273 +++ b/src/network-list.c
10274 @@ -104,6 +104,7 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
10275 loc_network_unref(list->elements[i]);
10277 free(list->elements);
10278 + list->elements = NULL;
10279 list->elements_size = 0;
10285 From 673e03f7bfc807248a769cb00ec80f0fa2af7a25 Mon Sep 17 00:00:00 2001
10286 From: Michael Tremer <michael.tremer@ipfire.org>
10287 Date: Wed, 25 Nov 2020 14:42:26 +0000
10288 Subject: [PATCH 099/111] network-list: Do not half list when popping the first
10291 The list was unfortunately halved in size every time an element
10292 was taken from it, which was great for performance, but shortened
10293 the result substantially.
10295 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10297 src/network-list.c | 5 ++++-
10298 1 file changed, 4 insertions(+), 1 deletion(-)
10300 diff --git a/src/network-list.c b/src/network-list.c
10301 index 2c4edb3..f4a9d05 100644
10302 --- a/src/network-list.c
10303 +++ b/src/network-list.c
10304 @@ -261,10 +261,13 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
10305 struct loc_network* network = list->elements[0];
10307 // Move all elements to the top of the stack
10308 - for (unsigned int i = 0; i < --list->size; i++) {
10309 + for (unsigned int i = 0; i < list->size - 1; i++) {
10310 list->elements[i] = list->elements[i+1];
10313 + // The list is shorter now
10316 DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
10322 From 9446c7538ab8b27486d74f6dc0dac8bb5c1bbe92 Mon Sep 17 00:00:00 2001
10323 From: Michael Tremer <michael.tremer@ipfire.org>
10324 Date: Wed, 25 Nov 2020 14:43:58 +0000
10325 Subject: [PATCH 100/111] network-list: Show index when listing networks
10327 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10329 src/network-list.c | 2 +-
10330 1 file changed, 1 insertion(+), 1 deletion(-)
10332 diff --git a/src/network-list.c b/src/network-list.c
10333 index f4a9d05..cf9459d 100644
10334 --- a/src/network-list.c
10335 +++ b/src/network-list.c
10336 @@ -119,7 +119,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
10338 s = loc_network_str(network);
10340 - INFO(list->ctx, "%s\n", s);
10341 + INFO(list->ctx, "%4d: %s\n", i, s);
10348 From dc31666be416dbb3bd2bc04127104a3f75b42d5c Mon Sep 17 00:00:00 2001
10349 From: Michael Tremer <michael.tremer@ipfire.org>
10350 Date: Wed, 25 Nov 2020 14:44:23 +0000
10351 Subject: [PATCH 101/111] network-list: Remove useless comment
10353 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10355 src/network-list.c | 2 --
10356 1 file changed, 2 deletions(-)
10358 diff --git a/src/network-list.c b/src/network-list.c
10359 index cf9459d..bd3d64e 100644
10360 --- a/src/network-list.c
10361 +++ b/src/network-list.c
10362 @@ -132,8 +132,6 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
10363 return loc_network_ref(list->elements[index]);
10366 -//MOVE FUNCTION GOES HERE
10368 static off_t loc_network_list_find(struct loc_network_list* list,
10369 struct loc_network* network, int* found) {
10374 From 82fa4c92c88cc172953198637bccc375c4d25d20 Mon Sep 17 00:00:00 2001
10375 From: Michael Tremer <michael.tremer@ipfire.org>
10376 Date: Wed, 25 Nov 2020 14:44:56 +0000
10377 Subject: [PATCH 102/111] networks: Add tests for overlaps function
10379 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10381 src/test-network.c | 10 ++++++++++
10382 1 file changed, 10 insertions(+)
10384 diff --git a/src/test-network.c b/src/test-network.c
10385 index 339743d..dde13f1 100644
10386 --- a/src/test-network.c
10387 +++ b/src/test-network.c
10388 @@ -192,6 +192,16 @@ int main(int argc, char** argv) {
10389 exit(EXIT_FAILURE);
10392 + if (!loc_network_overlaps(network1, subnet1)) {
10393 + fprintf(stderr, "Network1 does not seem to contain subnet1\n");
10394 + exit(EXIT_FAILURE);
10397 + if (!loc_network_overlaps(network1, subnet2)) {
10398 + fprintf(stderr, "Network1 does not seem to contain subnet2\n");
10399 + exit(EXIT_FAILURE);
10402 loc_network_unref(subnet1);
10403 loc_network_unref(subnet2);
10408 From 0a39d5499ae0c343a6600ea30de7d95e384ef911 Mon Sep 17 00:00:00 2001
10409 From: Michael Tremer <michael.tremer@ipfire.org>
10410 Date: Wed, 25 Nov 2020 15:11:21 +0000
10411 Subject: [PATCH 103/111] networks: Remove comparing family
10413 Everything is encoded in IPv6 anyways...
10415 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10417 src/network.c | 17 -----------------
10418 1 file changed, 17 deletions(-)
10420 diff --git a/src/network.c b/src/network.c
10421 index 394bafc..66b460f 100644
10422 --- a/src/network.c
10423 +++ b/src/network.c
10424 @@ -442,12 +442,6 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
10427 LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
10428 - // Compare family
10429 - if (self->family > other->family)
10431 - else if (self->family < other->family)
10435 int r = in6_addr_cmp(&self->first_address, &other->first_address);
10437 @@ -482,10 +476,6 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
10440 LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
10442 - if (self->family != other->family)
10445 // The prefix must be smaller (this avoids the more complex comparisons later)
10446 if (self->prefix > other->prefix)
10448 @@ -601,13 +591,6 @@ ERROR:
10450 static int __loc_network_exclude_to_list(struct loc_network* self,
10451 struct loc_network* other, struct loc_network_list* list) {
10452 - // Family must match
10453 - if (self->family != other->family) {
10454 - DEBUG(self->ctx, "Family mismatch\n");
10459 // Other must be a subnet of self
10460 if (!loc_network_is_subnet(self, other)) {
10461 DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
10465 From abf559267696061a0c9723d8f8e09c9d1aa8fb75 Mon Sep 17 00:00:00 2001
10466 From: Michael Tremer <michael.tremer@ipfire.org>
10467 Date: Wed, 25 Nov 2020 15:13:08 +0000
10468 Subject: [PATCH 104/111] network: Do not execute with an error when the
10469 excluded result will be empty
10471 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10473 src/network.c | 6 ++++--
10474 1 file changed, 4 insertions(+), 2 deletions(-)
10476 diff --git a/src/network.c b/src/network.c
10477 index 66b460f..9aa802f 100644
10478 --- a/src/network.c
10479 +++ b/src/network.c
10480 @@ -595,14 +595,16 @@ static int __loc_network_exclude_to_list(struct loc_network* self,
10481 if (!loc_network_is_subnet(self, other)) {
10482 DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
10489 // We cannot perform this operation if both networks equal
10490 if (loc_network_cmp(self, other) == 0) {
10491 DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
10498 return __loc_network_exclude(self, other, list);
10502 From 4fc034e20fdc36267f6ba18e27776fed747fe983 Mon Sep 17 00:00:00 2001
10503 From: Michael Tremer <michael.tremer@ipfire.org>
10504 Date: Wed, 25 Nov 2020 15:14:31 +0000
10505 Subject: [PATCH 105/111] network: Add more excluded networks straight to the
10508 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10510 src/network.c | 6 +-----
10511 1 file changed, 1 insertion(+), 5 deletions(-)
10513 diff --git a/src/network.c b/src/network.c
10514 index 9aa802f..19c387d 100644
10515 --- a/src/network.c
10516 +++ b/src/network.c
10517 @@ -705,11 +705,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
10518 if (loc_network_overlaps(subnet_to_check, subnet)) {
10521 - struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet);
10523 - loc_network_list_merge(to_check, excluded);
10524 - loc_network_list_unref(excluded);
10526 + __loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
10528 loc_network_unref(subnet);
10533 From 4de8ff8e5435225d3375a168054490d2a66b1baf Mon Sep 17 00:00:00 2001
10534 From: Michael Tremer <michael.tremer@ipfire.org>
10535 Date: Wed, 25 Nov 2020 15:15:33 +0000
10536 Subject: [PATCH 106/111] network: Call subnet function with the correct order
10539 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10541 src/network.c | 4 ++--
10542 1 file changed, 2 insertions(+), 2 deletions(-)
10544 diff --git a/src/network.c b/src/network.c
10545 index 19c387d..a96ce6d 100644
10546 --- a/src/network.c
10547 +++ b/src/network.c
10548 @@ -695,14 +695,14 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
10549 subnet = loc_network_list_get(list, i);
10551 // Drop this subnet if is a subnet of another subnet
10552 - if (loc_network_is_subnet(subnet_to_check, subnet)) {
10553 + if (loc_network_is_subnet(subnet, subnet_to_check)) {
10555 loc_network_unref(subnet);
10559 // Break it down if it overlaps
10560 - if (loc_network_overlaps(subnet_to_check, subnet)) {
10561 + if (loc_network_overlaps(subnet, subnet_to_check)) {
10564 __loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
10568 From 058e7800e56150bcf0fb8dceaae6fbab1ed239e4 Mon Sep 17 00:00:00 2001
10569 From: Michael Tremer <michael.tremer@ipfire.org>
10570 Date: Wed, 25 Nov 2020 15:16:06 +0000
10571 Subject: [PATCH 107/111] network: Massively improve performance on exclude
10573 When we check the result for any overlaps, we can cut this short
10574 by walking through both lists from start to end and remember the
10575 last network that we checked.
10577 The next one will by definition be strictly greater and therefore
10578 we do not need to check anything before this any more.
10580 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10582 src/network.c | 19 +++++++++++++++++--
10583 1 file changed, 17 insertions(+), 2 deletions(-)
10585 diff --git a/src/network.c b/src/network.c
10586 index a96ce6d..10fa997 100644
10587 --- a/src/network.c
10588 +++ b/src/network.c
10589 @@ -679,8 +679,10 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
10593 + off_t smallest_subnet = 0;
10595 while (!loc_network_list_empty(to_check)) {
10596 - struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
10597 + struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check);
10599 // Check whether the subnet to check is part of the input list
10600 if (loc_network_list_contains(list, subnet_to_check)) {
10601 @@ -691,7 +693,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
10602 // Marks whether this subnet passed all checks
10605 - for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
10606 + for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) {
10607 subnet = loc_network_list_get(list, i);
10609 // Drop this subnet if is a subnet of another subnet
10610 @@ -711,6 +713,19 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
10614 + // If the subnet is strictly greater, we do not need to continue the search
10615 + r = loc_network_cmp(subnet, subnet_to_check);
10617 + loc_network_unref(subnet);
10620 + // If it is strictly smaller, we can continue the search from here next
10621 + // time because all networks that are to be checked can only be larger
10622 + // than this one.
10623 + } else if (r < 0) {
10624 + smallest_subnet = i;
10627 loc_network_unref(subnet);
10633 From 9caf6cf5f0fd5eb56ad1678910dbc8017f48863c Mon Sep 17 00:00:00 2001
10634 From: Michael Tremer <michael.tremer@ipfire.org>
10635 Date: Wed, 25 Nov 2020 15:17:21 +0000
10636 Subject: [PATCH 108/111] network: Remove deprecated sort call
10638 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10640 src/network.c | 3 ---
10641 1 file changed, 3 deletions(-)
10643 diff --git a/src/network.c b/src/network.c
10644 index 10fa997..4d7ac79 100644
10645 --- a/src/network.c
10646 +++ b/src/network.c
10647 @@ -738,9 +738,6 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
10649 loc_network_list_unref(to_check);
10651 - // Sort the result
10652 - loc_network_list_sort(subnets);
10660 From 8ebf1d3912020d5c0ed98bb32a0640952bd6447c Mon Sep 17 00:00:00 2001
10661 From: Michael Tremer <michael.tremer@ipfire.org>
10662 Date: Wed, 25 Nov 2020 15:17:42 +0000
10663 Subject: [PATCH 109/111] network-list: Include network header
10665 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10667 src/loc/network-list.h | 2 ++
10668 1 file changed, 2 insertions(+)
10670 diff --git a/src/loc/network-list.h b/src/loc/network-list.h
10671 index 21c7402..bee21c4 100644
10672 --- a/src/loc/network-list.h
10673 +++ b/src/loc/network-list.h
10675 #ifndef LIBLOC_NETWORK_LIST_H
10676 #define LIBLOC_NETWORK_LIST_H
10678 +#include <loc/network.h>
10680 struct loc_network_list;
10681 int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
10682 struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
10686 From fff093b6827a31ff84a6d5150bfede140b696d25 Mon Sep 17 00:00:00 2001
10687 From: Michael Tremer <michael.tremer@ipfire.org>
10688 Date: Wed, 25 Nov 2020 15:24:43 +0000
10689 Subject: [PATCH 110/111] network-list: Use clear function to tidy up
10691 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10693 src/network-list.c | 4 ++--
10694 1 file changed, 2 insertions(+), 2 deletions(-)
10696 diff --git a/src/network-list.c b/src/network-list.c
10697 index bd3d64e..f3458b4 100644
10698 --- a/src/network-list.c
10699 +++ b/src/network-list.c
10700 @@ -70,8 +70,8 @@ LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list
10701 static void loc_network_list_free(struct loc_network_list* list) {
10702 DEBUG(list->ctx, "Releasing network list at %p\n", list);
10704 - for (unsigned int i = 0; i < list->size; i++)
10705 - loc_network_unref(list->elements[i]);
10706 + // Remove all content
10707 + loc_network_list_clear(list);
10709 loc_unref(list->ctx);
10714 From adbec5c0317a1c007a897fe3f63f0f86ff448124 Mon Sep 17 00:00:00 2001
10715 From: Michael Tremer <michael.tremer@ipfire.org>
10716 Date: Wed, 25 Nov 2020 20:00:46 +0000
10717 Subject: [PATCH 111/111] configure: Bump version to 0.9.5
10719 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
10721 configure.ac | 2 +-
10722 1 file changed, 1 insertion(+), 1 deletion(-)
10724 diff --git a/configure.ac b/configure.ac
10725 index 2364dfd..012d8ca 100644
10733 [location@lists.ipfire.org],
10735 [https://location.ipfire.org/])