libloc: Import changes from upstream

[people/pmueller/ipfire-2.x.git] / src / patches / libloc-0.9.4-upstream.patch

From ee6ea3986dc80183157f67275dc9f28231b5d5b2 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 24 Sep 2020 10:17:58 +0000
Subject: [PATCH 001/111] Revert "importer: Purge any redundant entries"

This reverts commit c2cc55d5a6875c3838f060032eaed89dcfb92ef6.

The query stalls the database and therefore the automatic
scripts are no longer able to generate a new version of the
database.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index 1467923..e3a07a0 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -374,27 +374,7 @@ class CLI(object):
 				INSERT INTO autnums(number, name)
 					SELECT _autnums.number, _organizations.name FROM _autnums
 						JOIN _organizations ON _autnums.organization = _organizations.handle
-				ON CONFLICT (number) DO UPDATE SET name = excluded.name
-			""")
-
-			self.db.execute("""
-				--- Purge any redundant entries
-				CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
-				AS
-				SELECT network FROM networks candidates
-				WHERE EXISTS (
-					SELECT FROM networks
-					WHERE
-						networks.network << candidates.network
-					AND
-						networks.country = candidates.country
-				);
-
-				CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
-
-				DELETE FROM networks WHERE EXISTS (
-					SELECT FROM _garbage WHERE networks.network = _garbage.network
-				);
+				ON CONFLICT (number) DO UPDATE SET name = excluded.name;
 			""")
 
 		# Download all extended sources
-- 
2.20.1

From 92f6abf4e272672bb0a71cfe991261b95ebe2fef Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 24 Sep 2020 10:18:58 +0000
Subject: [PATCH 002/111] Revert "importer: Import raw sources for inetnum's
 again"

This reverts commit 64e95fa903edec8b4e4e59830b395e2e4a411853.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/importer.py          | 14 ++++----
 src/python/location-importer.in | 63 ---------------------------------
 2 files changed, 7 insertions(+), 70 deletions(-)

diff --git a/src/python/importer.py b/src/python/importer.py
index f19db4b..de20f37 100644
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
 	"https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
 
 	# Asia Pacific Network Information Centre
-	"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-	"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+	#"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+	#"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
 	"https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
 	# XXX ???
 
 	# Réseaux IP Européens
-	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
 	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
 
 EXTENDED_SOURCES = (
 	# African Network Information Centre
-	#"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+	"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
 
 	# Asia Pacific Network Information Centre
-	#"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+	"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
 
 	# American Registry for Internet Numbers
 	"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
 	"http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
 
 	# Réseaux IP Européens
-	#"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+	"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
 )
 
 class Downloader(object):
diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index e3a07a0..77952f2 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -393,10 +393,6 @@ class CLI(object):
 		if line.startswith("aut-num:"):
 			return self._parse_autnum_block(block)
 
-		# inetnum
-		if line.startswith("inet6num:") or line.startswith("inetnum:"):
-			return self._parse_inetnum_block(block)
-
 		# organisation
 		elif line.startswith("organisation:"):
 			return self._parse_org_block(block)
@@ -426,65 +422,6 @@ class CLI(object):
 			autnum.get("asn"), autnum.get("org"),
 		)
 
-	def _parse_inetnum_block(self, block):
-		logging.debug("Parsing inetnum block:")
-
-		inetnum = {}
-		for line in block:
-			logging.debug(line)
-
-			# Split line
-			key, val = split_line(line)
-
-			if key == "inetnum":
-				start_address, delim, end_address = val.partition("-")
-
-				# Strip any excess space
-				start_address, end_address = start_address.rstrip(), end_address.strip()
-
-				# Convert to IP address
-				try:
-					start_address = ipaddress.ip_address(start_address)
-					end_address   = ipaddress.ip_address(end_address)
-				except ValueError:
-					logging.warning("Could not parse line: %s" % line)
-					return
-
-				# Set prefix to default
-				prefix = 32
-
-				# Count number of addresses in this subnet
-				num_addresses = int(end_address) - int(start_address)
-				if num_addresses:
-					prefix -= math.log(num_addresses, 2)
-
-				inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
-
-			elif key == "inet6num":
-				inetnum[key] = val
-
-			elif key == "country":
-				if val == "UNITED STATES":
-					val = "US"
-
-				inetnum[key] = val.upper()
-
-		# Skip empty objects
-		if not inetnum:
-			return
-
-		network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-
-		# Bail out in case we have processed a non-public IP network
-		if network.is_private:
-			logging.warning("Skipping non-globally routable network: %s" % network)
-			return
-
-		self.db.execute("INSERT INTO networks(network, country) \
-			VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
-			"%s" % network, inetnum.get("country"),
-		)
-
 	def _parse_org_block(self, block):
 		org = {}
 		for line in block:
-- 
2.20.1

From f532841e9197ce2f40aad8c086d786b2cb783a54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Mon, 12 Oct 2020 20:53:31 +0000
Subject: [PATCH 003/111] Revert "Revert "importer: Import raw sources for
 inetnum's again""

This reverts commit 92f6abf4e272672bb0a71cfe991261b95ebe2fef.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/importer.py          | 14 ++++----
 src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+), 7 deletions(-)

diff --git a/src/python/importer.py b/src/python/importer.py
index de20f37..f19db4b 100644
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
 	"https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
 
 	# Asia Pacific Network Information Centre
-	#"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-	#"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+	"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+	"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
 	"https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
 	# XXX ???
 
 	# Réseaux IP Européens
-	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
 	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
 
 EXTENDED_SOURCES = (
 	# African Network Information Centre
-	"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+	#"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
 
 	# Asia Pacific Network Information Centre
-	"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+	#"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
 
 	# American Registry for Internet Numbers
 	"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
 	"http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
 
 	# Réseaux IP Européens
-	"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+	#"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
 )
 
 class Downloader(object):
diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index 77952f2..e3a07a0 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -393,6 +393,10 @@ class CLI(object):
 		if line.startswith("aut-num:"):
 			return self._parse_autnum_block(block)
 
+		# inetnum
+		if line.startswith("inet6num:") or line.startswith("inetnum:"):
+			return self._parse_inetnum_block(block)
+
 		# organisation
 		elif line.startswith("organisation:"):
 			return self._parse_org_block(block)
@@ -422,6 +426,65 @@ class CLI(object):
 			autnum.get("asn"), autnum.get("org"),
 		)
 
+	def _parse_inetnum_block(self, block):
+		logging.debug("Parsing inetnum block:")
+
+		inetnum = {}
+		for line in block:
+			logging.debug(line)
+
+			# Split line
+			key, val = split_line(line)
+
+			if key == "inetnum":
+				start_address, delim, end_address = val.partition("-")
+
+				# Strip any excess space
+				start_address, end_address = start_address.rstrip(), end_address.strip()
+
+				# Convert to IP address
+				try:
+					start_address = ipaddress.ip_address(start_address)
+					end_address   = ipaddress.ip_address(end_address)
+				except ValueError:
+					logging.warning("Could not parse line: %s" % line)
+					return
+
+				# Set prefix to default
+				prefix = 32
+
+				# Count number of addresses in this subnet
+				num_addresses = int(end_address) - int(start_address)
+				if num_addresses:
+					prefix -= math.log(num_addresses, 2)
+
+				inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
+
+			elif key == "inet6num":
+				inetnum[key] = val
+
+			elif key == "country":
+				if val == "UNITED STATES":
+					val = "US"
+
+				inetnum[key] = val.upper()
+
+		# Skip empty objects
+		if not inetnum:
+			return
+
+		network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
+		# Bail out in case we have processed a non-public IP network
+		if network.is_private:
+			logging.warning("Skipping non-globally routable network: %s" % network)
+			return
+
+		self.db.execute("INSERT INTO networks(network, country) \
+			VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+			"%s" % network, inetnum.get("country"),
+		)
+
 	def _parse_org_block(self, block):
 		org = {}
 		for line in block:
-- 
2.20.1

From a36bc686865fc87ea386fd90b389338bdcb80cbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Mon, 12 Oct 2020 20:53:32 +0000
Subject: [PATCH 004/111] location-importer.in: only import relevant data from
 AFRINIC, APNIC and RIPE
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In contrast to ARIN and LACNIC, we are able to process more detailled
feeds from those RIRs, avoiding storage of obviously unnecessary data.

Thanks to various SQL optimisations, doing so now takes less time than
the first version of this did.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
 1 file changed, 87 insertions(+), 2 deletions(-)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index e3a07a0..093f325 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -165,6 +165,7 @@ class CLI(object):
 				-- networks
 				CREATE TABLE IF NOT EXISTS networks(network inet, country text);
 				CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
+				CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
 				CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
 
 				-- overrides
@@ -363,6 +364,16 @@ class CLI(object):
 				CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
 					ON COMMIT DROP;
 				CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
+
+				CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
+					ON COMMIT DROP;
+				CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
+				CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
+			""")
+
+			# Remove all previously imported content
+			self.db.execute("""
+				TRUNCATE TABLE networks;
 			""")
 
 			for source in location.importer.WHOIS_SOURCES:
@@ -370,6 +381,67 @@ class CLI(object):
 					for block in f:
 						self._parse_block(block)
 
+			# Process all parsed networks from every RIR we happen to have access to,
+			# insert the largest network chunks into the networks table immediately...
+			families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
+
+			for family in (row.family for row in families):
+				smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
+
+				self.db.execute("INSERT INTO networks(network, country) \
+					SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
+
+				# ... determine any other prefixes for this network family, ...
+				prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
+					WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
+
+				# ... and insert networks with this prefix in case they provide additional
+				# information (i. e. subnet of a larger chunk with a different country)
+				for prefix in (row.prefix for row in prefixes):
+					self.db.execute("""
+						WITH candidates AS (
+							SELECT
+								_rirdata.network,
+								_rirdata.country
+							FROM
+								_rirdata
+							WHERE
+								family(_rirdata.network) = %s
+							AND
+								masklen(_rirdata.network) = %s
+						),
+						filtered AS (
+							SELECT
+								DISTINCT ON (c.network)
+								c.network,
+								c.country,
+								masklen(networks.network),
+								networks.country AS parent_country
+							FROM
+								candidates c
+							LEFT JOIN
+								networks
+							ON
+								c.network << networks.network
+							ORDER BY
+								c.network,
+								masklen(networks.network) DESC NULLS LAST
+						)
+						INSERT INTO
+							networks(network, country)
+						SELECT
+							network,
+							country
+						FROM
+							filtered
+						WHERE
+							parent_country IS NULL
+						OR
+							country <> parent_country
+						ON CONFLICT DO NOTHING""",
+						family, prefix,
+					)
+
 			self.db.execute("""
 				INSERT INTO autnums(number, name)
 					SELECT _autnums.number, _organizations.name FROM _autnums
@@ -470,17 +542,30 @@ class CLI(object):
 				inetnum[key] = val.upper()
 
 		# Skip empty objects
-		if not inetnum:
+		if not inetnum or not "country" in inetnum:
 			return
 
 		network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
 
+		# Bail out in case we have processed a network covering the entire IP range, which
+		# is necessary to work around faulty (?) IPv6 network processing
+		if network.prefixlen == 0:
+			logging.warning("Skipping network covering the entire IP adress range: %s" % network)
+			return
+
+		# Bail out in case we have processed a network whose prefix length indicates it is
+		# not globally routable (we have decided not to process them at the moment, as they
+		# significantly enlarge our database without providing very helpful additional information)
+		if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
+			logging.info("Skipping network too small to be publicly announced: %s" % network)
+			return
+
 		# Bail out in case we have processed a non-public IP network
 		if network.is_private:
 			logging.warning("Skipping non-globally routable network: %s" % network)
 			return
 
-		self.db.execute("INSERT INTO networks(network, country) \
+		self.db.execute("INSERT INTO _rirdata(network, country) \
 			VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
 			"%s" % network, inetnum.get("country"),
 		)
-- 
2.20.1

From 2373de384f10f5573bbd7570f5522545df70c0e3 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 16 Oct 2020 12:24:58 +0000
Subject: [PATCH 005/111] location-importer: Include all overridden networks

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index 093f325..d249a35 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -240,6 +240,8 @@ class CLI(object):
 					SELECT network FROM announcements
 				UNION
 					SELECT network FROM networks
+				UNION
+					SELECT network FROM network_overrides
 				ORDER BY network
 			)
 
-- 
2.20.1

From 13f67f285856e8eabfeff2daf1be3aeaa36a82cc Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 16 Oct 2020 12:26:38 +0000
Subject: [PATCH 006/111] Revert "location-importer.in: only import relevant
 data from AFRINIC, APNIC and RIPE"

This reverts commit a36bc686865fc87ea386fd90b389338bdcb80cbc.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 89 +--------------------------------
 1 file changed, 2 insertions(+), 87 deletions(-)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index d249a35..b220eaf 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -165,7 +165,6 @@ class CLI(object):
 				-- networks
 				CREATE TABLE IF NOT EXISTS networks(network inet, country text);
 				CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
-				CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
 				CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
 
 				-- overrides
@@ -366,16 +365,6 @@ class CLI(object):
 				CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
 					ON COMMIT DROP;
 				CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
-
-				CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
-					ON COMMIT DROP;
-				CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
-				CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
-			""")
-
-			# Remove all previously imported content
-			self.db.execute("""
-				TRUNCATE TABLE networks;
 			""")
 
 			for source in location.importer.WHOIS_SOURCES:
@@ -383,67 +372,6 @@ class CLI(object):
 					for block in f:
 						self._parse_block(block)
 
-			# Process all parsed networks from every RIR we happen to have access to,
-			# insert the largest network chunks into the networks table immediately...
-			families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
-
-			for family in (row.family for row in families):
-				smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
-
-				self.db.execute("INSERT INTO networks(network, country) \
-					SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
-
-				# ... determine any other prefixes for this network family, ...
-				prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
-					WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
-
-				# ... and insert networks with this prefix in case they provide additional
-				# information (i. e. subnet of a larger chunk with a different country)
-				for prefix in (row.prefix for row in prefixes):
-					self.db.execute("""
-						WITH candidates AS (
-							SELECT
-								_rirdata.network,
-								_rirdata.country
-							FROM
-								_rirdata
-							WHERE
-								family(_rirdata.network) = %s
-							AND
-								masklen(_rirdata.network) = %s
-						),
-						filtered AS (
-							SELECT
-								DISTINCT ON (c.network)
-								c.network,
-								c.country,
-								masklen(networks.network),
-								networks.country AS parent_country
-							FROM
-								candidates c
-							LEFT JOIN
-								networks
-							ON
-								c.network << networks.network
-							ORDER BY
-								c.network,
-								masklen(networks.network) DESC NULLS LAST
-						)
-						INSERT INTO
-							networks(network, country)
-						SELECT
-							network,
-							country
-						FROM
-							filtered
-						WHERE
-							parent_country IS NULL
-						OR
-							country <> parent_country
-						ON CONFLICT DO NOTHING""",
-						family, prefix,
-					)
-
 			self.db.execute("""
 				INSERT INTO autnums(number, name)
 					SELECT _autnums.number, _organizations.name FROM _autnums
@@ -544,30 +472,17 @@ class CLI(object):
 				inetnum[key] = val.upper()
 
 		# Skip empty objects
-		if not inetnum or not "country" in inetnum:
+		if not inetnum:
 			return
 
 		network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
 
-		# Bail out in case we have processed a network covering the entire IP range, which
-		# is necessary to work around faulty (?) IPv6 network processing
-		if network.prefixlen == 0:
-			logging.warning("Skipping network covering the entire IP adress range: %s" % network)
-			return
-
-		# Bail out in case we have processed a network whose prefix length indicates it is
-		# not globally routable (we have decided not to process them at the moment, as they
-		# significantly enlarge our database without providing very helpful additional information)
-		if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
-			logging.info("Skipping network too small to be publicly announced: %s" % network)
-			return
-
 		# Bail out in case we have processed a non-public IP network
 		if network.is_private:
 			logging.warning("Skipping non-globally routable network: %s" % network)
 			return
 
-		self.db.execute("INSERT INTO _rirdata(network, country) \
+		self.db.execute("INSERT INTO networks(network, country) \
 			VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
 			"%s" % network, inetnum.get("country"),
 		)
-- 
2.20.1

From 44341478233115b26bb27fdb24da5b0a1eedb173 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 16 Oct 2020 12:26:43 +0000
Subject: [PATCH 007/111] Revert "Revert "Revert "importer: Import raw sources
 for inetnum's again"""

This reverts commit f532841e9197ce2f40aad8c086d786b2cb783a54.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/importer.py          | 14 ++++----
 src/python/location-importer.in | 63 ---------------------------------
 2 files changed, 7 insertions(+), 70 deletions(-)

diff --git a/src/python/importer.py b/src/python/importer.py
index f19db4b..de20f37 100644
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
 	"https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
 
 	# Asia Pacific Network Information Centre
-	"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-	"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+	#"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+	#"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
 	"https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
 	# XXX ???
 
 	# Réseaux IP Européens
-	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
 	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
 
 EXTENDED_SOURCES = (
 	# African Network Information Centre
-	#"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+	"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
 
 	# Asia Pacific Network Information Centre
-	#"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+	"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
 
 	# American Registry for Internet Numbers
 	"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
 	"http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
 
 	# Réseaux IP Européens
-	#"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+	"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
 )
 
 class Downloader(object):
diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index b220eaf..e87d378 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -395,10 +395,6 @@ class CLI(object):
 		if line.startswith("aut-num:"):
 			return self._parse_autnum_block(block)
 
-		# inetnum
-		if line.startswith("inet6num:") or line.startswith("inetnum:"):
-			return self._parse_inetnum_block(block)
-
 		# organisation
 		elif line.startswith("organisation:"):
 			return self._parse_org_block(block)
@@ -428,65 +424,6 @@ class CLI(object):
 			autnum.get("asn"), autnum.get("org"),
 		)
 
-	def _parse_inetnum_block(self, block):
-		logging.debug("Parsing inetnum block:")
-
-		inetnum = {}
-		for line in block:
-			logging.debug(line)
-
-			# Split line
-			key, val = split_line(line)
-
-			if key == "inetnum":
-				start_address, delim, end_address = val.partition("-")
-
-				# Strip any excess space
-				start_address, end_address = start_address.rstrip(), end_address.strip()
-
-				# Convert to IP address
-				try:
-					start_address = ipaddress.ip_address(start_address)
-					end_address   = ipaddress.ip_address(end_address)
-				except ValueError:
-					logging.warning("Could not parse line: %s" % line)
-					return
-
-				# Set prefix to default
-				prefix = 32
-
-				# Count number of addresses in this subnet
-				num_addresses = int(end_address) - int(start_address)
-				if num_addresses:
-					prefix -= math.log(num_addresses, 2)
-
-				inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
-
-			elif key == "inet6num":
-				inetnum[key] = val
-
-			elif key == "country":
-				if val == "UNITED STATES":
-					val = "US"
-
-				inetnum[key] = val.upper()
-
-		# Skip empty objects
-		if not inetnum:
-			return
-
-		network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-
-		# Bail out in case we have processed a non-public IP network
-		if network.is_private:
-			logging.warning("Skipping non-globally routable network: %s" % network)
-			return
-
-		self.db.execute("INSERT INTO networks(network, country) \
-			VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
-			"%s" % network, inetnum.get("country"),
-		)
-
 	def _parse_org_block(self, block):
 		org = {}
 		for line in block:
-- 
2.20.1

From a7d3a7a0565a0e09d3442e5829a0f30f016993b9 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 20 Oct 2020 20:44:43 +0000
Subject: [PATCH 008/111] as: Fix dereferencing NULL pointer when setting AS
 name

Reported-by: Gisle Vanem <gisle.vanem@gmail.com>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/as.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/as.c b/src/as.c
index e1fbb01..8421ac8 100644
--- a/src/as.c
+++ b/src/as.c
@@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
 }
 
 LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) {
-	as->name = strdup(name);
+	if (as->name)
+		free(as->name);
+
+	if (name)
+		as->name = strdup(name);
+	else
+		as->name = NULL;
 
 	return 0;
 }
-- 
2.20.1

From ddb326ad38a7c7202315dd2c6f938313db04ee22 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 21 Oct 2020 09:18:08 +0000
Subject: [PATCH 009/111] as: Do not attempt to match name when it wasn't set

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/as.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/as.c b/src/as.c
index 8421ac8..757bf3d 100644
--- a/src/as.c
+++ b/src/as.c
@@ -145,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
 	if (!string)
 		return 1;
 
+	// Cannot match anything when name is not set
+	if (!as->name)
+		return 1;
+
 	// Search if string is in name
 	if (strcasestr(as->name, string) != NULL)
 		return 1;
-- 
2.20.1

From d226ad2d97cbcd42ce807d9308569b1b9c5d4e2f Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 21 Oct 2020 09:28:39 +0000
Subject: [PATCH 010/111] writer: Free array with pointer to ASes, too

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/writer.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/writer.c b/src/writer.c
index 5939cff..160650f 100644
--- a/src/writer.c
+++ b/src/writer.c
@@ -147,8 +147,11 @@ static void loc_writer_free(struct loc_writer* writer) {
 		EVP_PKEY_free(writer->private_key2);
 
 	// Unref all AS
-	for (unsigned int i = 0; i < writer->as_count; i++) {
-		loc_as_unref(writer->as[i]);
+	if (writer->as) {
+		for (unsigned int i = 0; i < writer->as_count; i++) {
+			loc_as_unref(writer->as[i]);
+		}
+		free(writer->as);
 	}
 
 	// Release network tree
-- 
2.20.1

From d89a7d62772048ae1bd18d03f69df46b7e1a3d3c Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 21 Oct 2020 09:31:29 +0000
Subject: [PATCH 011/111] writer: Free countries when the writer is being
 destroyed

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/writer.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/writer.c b/src/writer.c
index 160650f..2f09b56 100644
--- a/src/writer.c
+++ b/src/writer.c
@@ -154,6 +154,14 @@ static void loc_writer_free(struct loc_writer* writer) {
 		free(writer->as);
 	}
 
+	// Unref all countries
+	if (writer->countries) {
+		for (unsigned int i = 0; i < writer->countries_count; i++) {
+			loc_country_unref(writer->countries[i]);
+		}
+		free(writer->countries);
+	}
+
 	// Release network tree
 	if (writer->networks)
 		loc_network_tree_unref(writer->networks);
-- 
2.20.1

From 0f1aedbc68e3945770c93e0ebd83eed0f555d6f0 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 21 Oct 2020 13:19:44 +0000
Subject: [PATCH 012/111] tests: Try adding an invalid network

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/test-network.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/test-network.c b/src/test-network.c
index d38f13d..e908b57 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -160,6 +160,14 @@ int main(int argc, char** argv) {
 	// Set ASN
 	loc_network_set_asn(network4, 1024);
 
+	// Try adding an invalid network
+	struct loc_network* network;
+	err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
+	if (err != -EINVAL) {
+		fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
+		exit(EXIT_FAILURE);
+	}
+
 	FILE* f = tmpfile();
 	if (!f) {
 		fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
-- 
2.20.1

From 13ad6e695f9ffc7847b3afe3e9cbcea8fb3a443f Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 21 Oct 2020 13:36:35 +0000
Subject: [PATCH 013/111] networks: Improve parsing IP addresses

loc_network_new_from_string() seem to have had some unexpected
behaviour for invalid inputs.

The function has been tidied up slightly and returns as soon as
some invalid input was detected.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c      | 52 +++++++++++++++++++++++++++++++---------------
 src/test-network.c |  7 +++++++
 2 files changed, 42 insertions(+), 17 deletions(-)

diff --git a/src/network.c b/src/network.c
index 366caa2..c112a41 100644
--- a/src/network.c
+++ b/src/network.c
@@ -160,9 +160,10 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
 LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
 		const char* address_string) {
 	struct in6_addr first_address;
-	unsigned int prefix = 0;
 	char* prefix_string;
-	int r = 1;
+	int r = -EINVAL;
+
+	DEBUG(ctx, "Attempting to parse network %s\n", address_string);
 
 	// Make a copy of the string to work on it
 	char* buffer = strdup(address_string);
@@ -171,29 +172,46 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
 	// Split address and prefix
 	address_string = strsep(&prefix_string, "/");
 
-	// Did we find a prefix?
-	if (prefix_string) {
-		// Convert prefix to integer
-		prefix = strtol(prefix_string, NULL, 10);
+	DEBUG(ctx, "  Split into address = %s, prefix = %s\n", address_string, prefix_string);
 
-		if (prefix) {
-			// Parse the address
-			r = loc_parse_address(ctx, address_string, &first_address);
+	// We need to have a prefix
+	if (!prefix_string) {
+		DEBUG(ctx, "No prefix set\n");
+		goto FAIL;
+	}
 
-			// Map the prefix to IPv6 if needed
-			if (IN6_IS_ADDR_V4MAPPED(&first_address))
-				prefix += 96;
-		}
+	// Convert prefix to integer
+	unsigned int prefix = strtol(prefix_string, NULL, 10);
+
+	// End if the prefix was invalid
+	if (!prefix) {
+		DEBUG(ctx, "The prefix is zero or not a number\n");
+		goto FAIL;
+	}
+
+	// Parse the address
+	r = loc_parse_address(ctx, address_string, &first_address);
+	if (r) {
+		DEBUG(ctx, "The address could not be parsed\n");
+		goto FAIL;
 	}
 
+	// Map the prefix to IPv6 if needed
+	if (IN6_IS_ADDR_V4MAPPED(&first_address))
+		prefix += 96;
+
+FAIL:
 	// Free temporary buffer
 	free(buffer);
 
-	if (r == 0) {
-		r = loc_network_new(ctx, network, &first_address, prefix);
-	}
+	// Exit if the parsing was unsuccessful
+	if (r)
+		return r;
+
+	DEBUG(ctx, "GOT HERE\n");
 
-	return r;
+	// Create a new network
+	return loc_network_new(ctx, network, &first_address, prefix);
 }
 
 LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
diff --git a/src/test-network.c b/src/test-network.c
index e908b57..85eca00 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -168,6 +168,13 @@ int main(int argc, char** argv) {
 		exit(EXIT_FAILURE);
 	}
 
+	// Try adding a single address
+	err = loc_writer_add_network(writer, &network, "2001:db8::");
+	if (err != -EINVAL) {
+		fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
+		exit(EXIT_FAILURE);
+	}
+
 	FILE* f = tmpfile();
 	if (!f) {
 		fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
-- 
2.20.1

From 6a467e9345bb5a3d37911c9aaac30019eaa4492b Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 21 Oct 2020 13:43:21 +0000
Subject: [PATCH 014/111] networks: Test if we can add localhost (IPv6)

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/test-network.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/test-network.c b/src/test-network.c
index 85eca00..8c7e898 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -175,6 +175,13 @@ int main(int argc, char** argv) {
 		exit(EXIT_FAILURE);
 	}
 
+	// Try adding localhost
+	err = loc_writer_add_network(writer, &network, "::1/128");
+	if (err != -EINVAL) {
+		fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
+		exit(EXIT_FAILURE);
+	}
+
 	FILE* f = tmpfile();
 	if (!f) {
 		fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
-- 
2.20.1

From fc1190aa11e3ff3d2dbf5f4d408c298e7916f46f Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 21 Oct 2020 13:44:50 +0000
Subject: [PATCH 015/111] networks: Remove accidentially committed debug line

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/network.c b/src/network.c
index c112a41..be88d75 100644
--- a/src/network.c
+++ b/src/network.c
@@ -208,8 +208,6 @@ FAIL:
 	if (r)
 		return r;
 
-	DEBUG(ctx, "GOT HERE\n");
-
 	// Create a new network
 	return loc_network_new(ctx, network, &first_address, prefix);
 }
-- 
2.20.1

From a1707d8983898b6878cdd5c68744bcc444e278ed Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 21 Oct 2020 13:53:36 +0000
Subject: [PATCH 016/111] importer: Add search index to announcements table

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index e87d378..d0fe5a6 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -152,6 +152,7 @@ class CLI(object):
 					last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
 				CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
 				CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
+				CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
 
 				-- autnums
 				CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
-- 
2.20.1

From 991baf530d47adb2ed7a15b65dc4565d07fa6d07 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 21 Oct 2020 13:54:45 +0000
Subject: [PATCH 017/111] importer: Add search index to network_overrides table

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index d0fe5a6..fe21d73 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -189,6 +189,8 @@ class CLI(object):
 				);
 				CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
 					ON network_overrides(network);
+				CREATE INDEX IF NOT EXISTS network_overrides_search
+					ON network_overrides USING GIST(network inet_ops);
 			""")
 
 		return db
-- 
2.20.1

From bbea93a74651df10e2ffdbd09eb434dc6a0471bc Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 21 Oct 2020 16:01:57 +0000
Subject: [PATCH 018/111] importer: Restructure SQL query to be executed in
 parallel

There are no functional changes, this just runs quicker now.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 67 ++++++++++++++++++---------------
 1 file changed, 37 insertions(+), 30 deletions(-)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index fe21d73..3c1e5e2 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -237,34 +237,24 @@ class CLI(object):
 
 		# Select all known networks
 		rows = self.db.query("""
-			-- Get a (sorted) list of all known networks
-			WITH known_networks AS (
-					SELECT network FROM announcements
-				UNION
-					SELECT network FROM networks
-				UNION
-					SELECT network FROM network_overrides
-				ORDER BY network
-			)
-
 			-- Return a list of those networks enriched with all
 			-- other information that we store in the database
 			SELECT
-				DISTINCT ON (known_networks.network)
-				known_networks.network AS network,
-				announcements.autnum AS autnum,
+				DISTINCT ON (network)
+				network,
+				autnum,
 
 				-- Country
 				COALESCE(
 					(
 						SELECT country FROM network_overrides overrides
-							WHERE announcements.network <<= overrides.network
+							WHERE networks.network <<= overrides.network
 							ORDER BY masklen(overrides.network) DESC
 							LIMIT 1
 					),
 					(
 						SELECT country FROM autnum_overrides overrides
-							WHERE announcements.autnum = overrides.number
+							WHERE networks.autnum = overrides.number
 					),
 					networks.country
 				) AS country,
@@ -273,50 +263,67 @@ class CLI(object):
 				COALESCE(
 					(
 						SELECT is_anonymous_proxy FROM network_overrides overrides
-							WHERE announcements.network <<= overrides.network
+							WHERE networks.network <<= overrides.network
 							ORDER BY masklen(overrides.network) DESC
 							LIMIT 1
 					),
 					(
 						SELECT is_anonymous_proxy FROM autnum_overrides overrides
-							WHERE announcements.autnum = overrides.number
+							WHERE networks.autnum = overrides.number
 					),
 					FALSE
 				) AS is_anonymous_proxy,
 				COALESCE(
 					(
 						SELECT is_satellite_provider FROM network_overrides overrides
-							WHERE announcements.network <<= overrides.network
+							WHERE networks.network <<= overrides.network
 							ORDER BY masklen(overrides.network) DESC
 							LIMIT 1
 					),
 					(
 						SELECT is_satellite_provider FROM autnum_overrides overrides
-							WHERE announcements.autnum = overrides.number
+							WHERE networks.autnum = overrides.number
 					),
 					FALSE
 				) AS is_satellite_provider,
 				COALESCE(
 					(
 						SELECT is_anycast FROM network_overrides overrides
-							WHERE announcements.network <<= overrides.network
+							WHERE networks.network <<= overrides.network
 							ORDER BY masklen(overrides.network) DESC
 							LIMIT 1
 					),
 					(
 						SELECT is_anycast FROM autnum_overrides overrides
-							WHERE announcements.autnum = overrides.number
+							WHERE networks.autnum = overrides.number
 					),
 					FALSE
-				) AS is_anycast,
-
-				-- Must be part of returned values for ORDER BY clause
-				masklen(announcements.network) AS sort_a,
-				masklen(networks.network) AS sort_b
-			FROM known_networks
-				LEFT JOIN announcements ON known_networks.network <<= announcements.network
-				LEFT JOIN networks ON known_networks.network <<= networks.network
-			ORDER BY known_networks.network, sort_a DESC, sort_b DESC
+				) AS is_anycast
+			FROM (
+				SELECT
+					known_networks.network AS network,
+					announcements.autnum AS autnum,
+					networks.country AS country,
+
+					-- Must be part of returned values for ORDER BY clause
+					masklen(announcements.network) AS sort_a,
+					masklen(networks.network) AS sort_b
+				FROM (
+						SELECT network FROM announcements
+					UNION ALL
+						SELECT network FROM networks
+					UNION ALL
+						SELECT network FROM network_overrides
+					) known_networks
+				LEFT JOIN
+					announcements ON known_networks.network <<= announcements.network
+				LEFT JOIN
+					networks ON known_networks.network <<= networks.network
+				ORDER BY
+					known_networks.network,
+					sort_a DESC,
+					sort_b DESC
+			) networks
 		""")
 
 		for row in rows:
-- 
2.20.1

From 26ab419b68d166f932db1f97c38cb9d793d04187 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 22 Oct 2020 12:24:34 +0000
Subject: [PATCH 019/111] network: Allow adding single IP addresses and
 automatically add the prefix

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c      | 33 +++++++++++++++------------------
 src/test-network.c |  4 ++--
 2 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/src/network.c b/src/network.c
index be88d75..d7b1645 100644
--- a/src/network.c
+++ b/src/network.c
@@ -161,6 +161,7 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
 		const char* address_string) {
 	struct in6_addr first_address;
 	char* prefix_string;
+	unsigned int prefix = 128;
 	int r = -EINVAL;
 
 	DEBUG(ctx, "Attempting to parse network %s\n", address_string);
@@ -174,21 +175,6 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
 
 	DEBUG(ctx, "  Split into address = %s, prefix = %s\n", address_string, prefix_string);
 
-	// We need to have a prefix
-	if (!prefix_string) {
-		DEBUG(ctx, "No prefix set\n");
-		goto FAIL;
-	}
-
-	// Convert prefix to integer
-	unsigned int prefix = strtol(prefix_string, NULL, 10);
-
-	// End if the prefix was invalid
-	if (!prefix) {
-		DEBUG(ctx, "The prefix is zero or not a number\n");
-		goto FAIL;
-	}
-
 	// Parse the address
 	r = loc_parse_address(ctx, address_string, &first_address);
 	if (r) {
@@ -196,9 +182,20 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
 		goto FAIL;
 	}
 
-	// Map the prefix to IPv6 if needed
-	if (IN6_IS_ADDR_V4MAPPED(&first_address))
-		prefix += 96;
+	// If a prefix was given, we will try to parse it
+	if (prefix_string) {
+		// Convert prefix to integer
+		prefix = strtol(prefix_string, NULL, 10);
+
+		if (!prefix) {
+			DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
+			goto FAIL;
+		}
+
+		// Map the prefix to IPv6 if needed
+		if (IN6_IS_ADDR_V4MAPPED(&first_address))
+			prefix += 96;
+	}
 
 FAIL:
 	// Free temporary buffer
diff --git a/src/test-network.c b/src/test-network.c
index 8c7e898..b6776b4 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -170,8 +170,8 @@ int main(int argc, char** argv) {
 
 	// Try adding a single address
 	err = loc_writer_add_network(writer, &network, "2001:db8::");
-	if (err != -EINVAL) {
-		fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
+	if (err) {
+		fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
 		exit(EXIT_FAILURE);
 	}
 
-- 
2.20.1

From aadac4c569e921be1d28dd3b2377ac7f3732213e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Wed, 21 Oct 2020 14:47:36 +0000
Subject: [PATCH 020/111] Revert "Revert "Revert "Revert "importer: Import raw
 sources for inetnum's again""""
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 44341478233115b26bb27fdb24da5b0a1eedb173.

Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/importer.py          | 14 ++++----
 src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+), 7 deletions(-)

diff --git a/src/python/importer.py b/src/python/importer.py
index de20f37..f19db4b 100644
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
 	"https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
 
 	# Asia Pacific Network Information Centre
-	#"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-	#"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+	"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+	"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
 	"https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
 	# XXX ???
 
 	# Réseaux IP Européens
-	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
 	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
 
 EXTENDED_SOURCES = (
 	# African Network Information Centre
-	"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+	#"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
 
 	# Asia Pacific Network Information Centre
-	"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+	#"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
 
 	# American Registry for Internet Numbers
 	"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
 	"http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
 
 	# Réseaux IP Européens
-	"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+	#"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
 )
 
 class Downloader(object):
diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index 3c1e5e2..e8a4fc5 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -405,6 +405,10 @@ class CLI(object):
 		if line.startswith("aut-num:"):
 			return self._parse_autnum_block(block)
 
+		# inetnum
+		if line.startswith("inet6num:") or line.startswith("inetnum:"):
+			return self._parse_inetnum_block(block)
+
 		# organisation
 		elif line.startswith("organisation:"):
 			return self._parse_org_block(block)
@@ -434,6 +438,65 @@ class CLI(object):
 			autnum.get("asn"), autnum.get("org"),
 		)
 
+	def _parse_inetnum_block(self, block):
+		logging.debug("Parsing inetnum block:")
+
+		inetnum = {}
+		for line in block:
+			logging.debug(line)
+
+			# Split line
+			key, val = split_line(line)
+
+			if key == "inetnum":
+				start_address, delim, end_address = val.partition("-")
+
+				# Strip any excess space
+				start_address, end_address = start_address.rstrip(), end_address.strip()
+
+				# Convert to IP address
+				try:
+					start_address = ipaddress.ip_address(start_address)
+					end_address   = ipaddress.ip_address(end_address)
+				except ValueError:
+					logging.warning("Could not parse line: %s" % line)
+					return
+
+				# Set prefix to default
+				prefix = 32
+
+				# Count number of addresses in this subnet
+				num_addresses = int(end_address) - int(start_address)
+				if num_addresses:
+					prefix -= math.log(num_addresses, 2)
+
+				inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
+
+			elif key == "inet6num":
+				inetnum[key] = val
+
+			elif key == "country":
+				if val == "UNITED STATES":
+					val = "US"
+
+				inetnum[key] = val.upper()
+
+		# Skip empty objects
+		if not inetnum:
+			return
+
+		network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
+		# Bail out in case we have processed a non-public IP network
+		if network.is_private:
+			logging.warning("Skipping non-globally routable network: %s" % network)
+			return
+
+		self.db.execute("INSERT INTO networks(network, country) \
+			VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+			"%s" % network, inetnum.get("country"),
+		)
+
 	def _parse_org_block(self, block):
 		org = {}
 		for line in block:
-- 
2.20.1

From 002deb6b42ac0b3624c07e3352cebd72dc0685a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Wed, 21 Oct 2020 14:47:37 +0000
Subject: [PATCH 021/111] Revert "Revert "location-importer.in: only import
 relevant data from AFRINIC, APNIC and RIPE""
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 13f67f285856e8eabfeff2daf1be3aeaa36a82cc.

Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
 1 file changed, 87 insertions(+), 2 deletions(-)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index e8a4fc5..5656c41 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -166,6 +166,7 @@ class CLI(object):
 				-- networks
 				CREATE TABLE IF NOT EXISTS networks(network inet, country text);
 				CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
+				CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
 				CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
 
 				-- overrides
@@ -375,6 +376,16 @@ class CLI(object):
 				CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
 					ON COMMIT DROP;
 				CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
+
+				CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
+					ON COMMIT DROP;
+				CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
+				CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
+			""")
+
+			# Remove all previously imported content
+			self.db.execute("""
+				TRUNCATE TABLE networks;
 			""")
 
 			for source in location.importer.WHOIS_SOURCES:
@@ -382,6 +393,67 @@ class CLI(object):
 					for block in f:
 						self._parse_block(block)
 
+			# Process all parsed networks from every RIR we happen to have access to,
+			# insert the largest network chunks into the networks table immediately...
+			families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
+
+			for family in (row.family for row in families):
+				smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
+
+				self.db.execute("INSERT INTO networks(network, country) \
+					SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
+
+				# ... determine any other prefixes for this network family, ...
+				prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
+					WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
+
+				# ... and insert networks with this prefix in case they provide additional
+				# information (i. e. subnet of a larger chunk with a different country)
+				for prefix in (row.prefix for row in prefixes):
+					self.db.execute("""
+						WITH candidates AS (
+							SELECT
+								_rirdata.network,
+								_rirdata.country
+							FROM
+								_rirdata
+							WHERE
+								family(_rirdata.network) = %s
+							AND
+								masklen(_rirdata.network) = %s
+						),
+						filtered AS (
+							SELECT
+								DISTINCT ON (c.network)
+								c.network,
+								c.country,
+								masklen(networks.network),
+								networks.country AS parent_country
+							FROM
+								candidates c
+							LEFT JOIN
+								networks
+							ON
+								c.network << networks.network
+							ORDER BY
+								c.network,
+								masklen(networks.network) DESC NULLS LAST
+						)
+						INSERT INTO
+							networks(network, country)
+						SELECT
+							network,
+							country
+						FROM
+							filtered
+						WHERE
+							parent_country IS NULL
+						OR
+							country <> parent_country
+						ON CONFLICT DO NOTHING""",
+						family, prefix,
+					)
+
 			self.db.execute("""
 				INSERT INTO autnums(number, name)
 					SELECT _autnums.number, _organizations.name FROM _autnums
@@ -482,17 +554,30 @@ class CLI(object):
 				inetnum[key] = val.upper()
 
 		# Skip empty objects
-		if not inetnum:
+		if not inetnum or not "country" in inetnum:
 			return
 
 		network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
 
+		# Bail out in case we have processed a network covering the entire IP range, which
+		# is necessary to work around faulty (?) IPv6 network processing
+		if network.prefixlen == 0:
+			logging.warning("Skipping network covering the entire IP adress range: %s" % network)
+			return
+
+		# Bail out in case we have processed a network whose prefix length indicates it is
+		# not globally routable (we have decided not to process them at the moment, as they
+		# significantly enlarge our database without providing very helpful additional information)
+		if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
+			logging.info("Skipping network too small to be publicly announced: %s" % network)
+			return
+
 		# Bail out in case we have processed a non-public IP network
 		if network.is_private:
 			logging.warning("Skipping non-globally routable network: %s" % network)
 			return
 
-		self.db.execute("INSERT INTO networks(network, country) \
+		self.db.execute("INSERT INTO _rirdata(network, country) \
 			VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
 			"%s" % network, inetnum.get("country"),
 		)
-- 
2.20.1

From 28c73fa3f4257e0a41e52af8a9643da414a6cb6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Wed, 21 Oct 2020 14:47:38 +0000
Subject: [PATCH 022/111] export.py: fix exporting IP networks for crappy
 xt_geoip module
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In contrast to the location database itself, the xt_geoip module
consumes a list of IP networks for each country, and returns after the
first match.

We therefore need to...

(a) sort IP networks by their size, allow as precise matches as possible
(b) export _any_ IP networks - including inverted subnets - to prevent
    undefined overlaps
(c) do the entire thing as fast as possible, consuming as less disk
    space as possible, which is why we can't just iterate over all /24
    chunks

Partially fixes: #12499

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/export.py | 69 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 54 insertions(+), 15 deletions(-)

diff --git a/src/python/export.py b/src/python/export.py
index d15c6f0..5eaf43f 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -39,8 +39,8 @@ class OutputWriter(object):
 	suffix = "networks"
 	mode = "w"
 
-	def __init__(self, f, prefix=None, flatten=True):
-		self.f, self.prefix, self.flatten = f, prefix, flatten
+	def __init__(self, db, f, prefix=None, flatten=True):
+		self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten
 
 		# The previously written network
 		self._last_network = None
@@ -49,13 +49,13 @@ class OutputWriter(object):
 		self._write_header()
 
 	@classmethod
-	def open(cls, filename, **kwargs):
+	def open(cls, db, filename, **kwargs):
 		"""
 			Convenience function to open a file
 		"""
 		f = open(filename, cls.mode)
 
-		return cls(f, **kwargs)
+		return cls(db, f, **kwargs)
 
 	def __repr__(self):
 		return "<%s f=%s>" % (self.__class__.__name__, self.f)
@@ -87,13 +87,31 @@ class OutputWriter(object):
 	def _write_network(self, network):
 		self.f.write("%s\n" % network)
 
-	def write(self, network):
+	def write(self, network, subnets):
 		if self.flatten and self._flatten(network):
 			log.debug("Skipping writing network %s" % network)
 			return
 
-		# Write the network to file
-		self._write_network(network)
+		# Write the network when it has no subnets
+		if not subnets:
+			network = ipaddress.ip_network("%s" % network)
+			return self._write_network(network)
+
+		# Collect all matching subnets
+		matching_subnets = []
+
+		for subnet in sorted(subnets):
+			# Try to find the subnet in the database
+			n = self.db.lookup("%s" % subnet.network_address)
+
+			# No entry or matching country means those IP addresses belong here
+			if not n or n.country_code == network.country_code:
+				matching_subnets.append(subnet)
+
+		# Write all networks as compact as possible
+		for network in ipaddress.collapse_addresses(matching_subnets):
+			log.debug("Writing %s to database" % network)
+			self._write_network(network)
 
 	def finish(self):
 		"""
@@ -143,10 +161,10 @@ class XTGeoIPOutputWriter(OutputWriter):
 	mode = "wb"
 
 	def _write_network(self, network):
-		for address in (network.first_address, network.last_address):
+		for address in (network.network_address, network.broadcast_address):
 			# Convert this into a string of bits
 			bytes = socket.inet_pton(
-				network.family, address,
+				socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
 			)
 
 			self.f.write(bytes)
@@ -175,7 +193,7 @@ class Exporter(object):
 					directory, prefix=country_code, suffix=self.writer.suffix, family=family,
 				)
 
-				writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
+				writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code)
 
 			# Create writers for ASNs
 			for asn in asns:
@@ -183,22 +201,43 @@ class Exporter(object):
 					directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
 				)
 
-				writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
+				writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
 
 			# Get all networks that match the family
 			networks = self.db.search_networks(family=family)
 
+			# Materialise the generator into a list (uses quite some memory)
+			networks = list(networks)
+
 			# Walk through all networks
-			for network in networks:
+			for i, network in enumerate(networks):
+				_network = ipaddress.ip_network("%s" % network)
+
+				# Search for all subnets
+				subnets = set()
+
+				while i < len(networks):
+					subnet = networks[i+1]
+
+					if subnet.is_subnet_of(network):
+						_subnet = ipaddress.ip_network("%s" % subnet)
+
+						subnets.add(_subnet)
+						subnets.update(_network.address_exclude(_subnet))
+
+						i += 1
+					else:
+						break
+
 				# Write matching countries
 				try:
-					writers[network.country_code].write(network)
+					writers[network.country_code].write(network, subnets)
 				except KeyError:
 					pass
 
 				# Write matching ASNs
 				try:
-					writers[network.asn].write(network)
+					writers[network.asn].write(network, subnets)
 				except KeyError:
 					pass
 
@@ -209,7 +248,7 @@ class Exporter(object):
 						country = flags[flag]
 
 						try:
-							writers[country].write(network)
+							writers[country].write(network, subnets)
 						except KeyError:
 							pass
 
-- 
2.20.1

From bd341642fc6bbcc050e9b4ec5124585c83cab84d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Wed, 21 Oct 2020 14:47:39 +0000
Subject: [PATCH 023/111] location-importer.in: filter bogus IP networks for
 both Whois and extended sources
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Sanity checks for parsed networks have been put into a separate function
to avoid boilerplate code for extended sources. This makes the location
database less vulnerable to garbage written into RIR databases on
purpose or by chance.

Fixes: #12500

Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 83 ++++++++++++++++++++++++++-------
 1 file changed, 67 insertions(+), 16 deletions(-)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index 5656c41..f24d357 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -469,6 +469,69 @@ class CLI(object):
 					for line in f:
 						self._parse_line(line)
 
+	def _check_parsed_network(self, network):
+		"""
+			Assistive function to detect and subsequently sort out parsed
+			networks from RIR data (both Whois and so-called "extended sources"),
+			which are or have...
+
+			(a) not globally routable (RFC 1918 space, et al.)
+			(b) covering a too large chunk of the IP address space (prefix length
+				is < 7 for IPv4 networks, and < 10 for IPv6)
+			(c) "0.0.0.0" or "::" as a network address
+			(d) are too small for being publicly announced (we have decided not to
+				process them at the moment, as they significantly enlarge our
+				database without providing very helpful additional information)
+
+			This unfortunately is necessary due to brain-dead clutter across
+			various RIR databases, causing mismatches and eventually disruptions.
+
+			We will return False in case a network is not suitable for adding
+			it to our database, and True otherwise.
+		"""
+
+		if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
+			return False
+
+		if not network.is_global:
+			logging.warning("Skipping non-globally routable network: %s" % network)
+			return False
+
+		if network.version == 4:
+			if network.prefixlen < 7:
+				logging.warning("Skipping too big IP chunk: %s" % network)
+				return False
+
+			if network.prefixlen > 24:
+				logging.info("Skipping network too small to be publicly announced: %s" % network)
+				return False
+
+			if str(network.network_address) == "0.0.0.0":
+				logging.warning("Skipping network based on 0.0.0.0: %s" % network)
+				return False
+
+		elif network.version == 6:
+			if network.prefixlen < 10:
+				logging.warning("Skipping too big IP chunk: %s" % network)
+				return False
+
+			if network.prefixlen > 48:
+				logging.info("Skipping network too small to be publicly announced: %s" % network)
+				return False
+
+			if str(network.network_address) == "::":
+				logging.warning("Skipping network based on '::': %s" % network)
+				return False
+
+		else:
+			# This should not happen...
+			logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
+			return False
+
+		# In case we have made it here, the network is considered to
+		# be suitable for libloc consumption...
+		return True
+
 	def _parse_block(self, block):
 		# Get first line to find out what type of block this is
 		line = block[0]
@@ -559,22 +622,7 @@ class CLI(object):
 
 		network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
 
-		# Bail out in case we have processed a network covering the entire IP range, which
-		# is necessary to work around faulty (?) IPv6 network processing
-		if network.prefixlen == 0:
-			logging.warning("Skipping network covering the entire IP adress range: %s" % network)
-			return
-
-		# Bail out in case we have processed a network whose prefix length indicates it is
-		# not globally routable (we have decided not to process them at the moment, as they
-		# significantly enlarge our database without providing very helpful additional information)
-		if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
-			logging.info("Skipping network too small to be publicly announced: %s" % network)
-			return
-
-		# Bail out in case we have processed a non-public IP network
-		if network.is_private:
-			logging.warning("Skipping non-globally routable network: %s" % network)
+		if not self._check_parsed_network(network):
 			return
 
 		self.db.execute("INSERT INTO _rirdata(network, country) \
@@ -658,6 +706,9 @@ class CLI(object):
 			log.warning("Invalid IP address: %s" % address)
 			return
 
+		if not self._check_parsed_network(network):
+			return
+
 		self.db.execute("INSERT INTO networks(network, country) \
 			VALUES(%s, %s) ON CONFLICT (network) DO \
 			UPDATE SET country = excluded.country",
-- 
2.20.1

From eee65490a10e0fe89b3834b8be176fc900084fa0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Wed, 21 Oct 2020 14:47:40 +0000
Subject: [PATCH 024/111] importer.py: fetch LACNIC data via HTTPS
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/importer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/python/importer.py b/src/python/importer.py
index f19db4b..5f46bc3 100644
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
 	"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
 
 	# Latin America and Caribbean Network Information Centre
-	"http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
+	"https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
 
 	# Réseaux IP Européens
 	#"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
-- 
2.20.1

From 84187ab5436eb158529d6f5e2a38890b4af3ddb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Wed, 21 Oct 2020 14:47:41 +0000
Subject: [PATCH 025/111] location-importer.in: omit historic/orphaned RIR data
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Some RIRs include detailled information regarding networks not managed
by or allocated to themselves, particually APNIC. We need to filter
those networks (they usually have a characteristic network name) in
order to prevent operational quirks or returning wrong country codes.

Fixes: #12501
Partially fixes: #12499

Cc: Michael Tremer <michael.tremer@ipfire.org>
Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 38 +++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index f24d357..a869256 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -494,38 +494,38 @@ class CLI(object):
 			return False
 
 		if not network.is_global:
-			logging.warning("Skipping non-globally routable network: %s" % network)
+			log.warning("Skipping non-globally routable network: %s" % network)
 			return False
 
 		if network.version == 4:
 			if network.prefixlen < 7:
-				logging.warning("Skipping too big IP chunk: %s" % network)
+				log.warning("Skipping too big IP chunk: %s" % network)
 				return False
 
 			if network.prefixlen > 24:
-				logging.info("Skipping network too small to be publicly announced: %s" % network)
+				log.info("Skipping network too small to be publicly announced: %s" % network)
 				return False
 
 			if str(network.network_address) == "0.0.0.0":
-				logging.warning("Skipping network based on 0.0.0.0: %s" % network)
+				log.warning("Skipping network based on 0.0.0.0: %s" % network)
 				return False
 
 		elif network.version == 6:
 			if network.prefixlen < 10:
-				logging.warning("Skipping too big IP chunk: %s" % network)
+				log.warning("Skipping too big IP chunk: %s" % network)
 				return False
 
 			if network.prefixlen > 48:
-				logging.info("Skipping network too small to be publicly announced: %s" % network)
+				log.info("Skipping network too small to be publicly announced: %s" % network)
 				return False
 
 			if str(network.network_address) == "::":
-				logging.warning("Skipping network based on '::': %s" % network)
+				log.warning("Skipping network based on '::': %s" % network)
 				return False
 
 		else:
 			# This should not happen...
-			logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
+			log.warning("Skipping network of unknown family, this should not happen: %s" % network)
 			return False
 
 		# In case we have made it here, the network is considered to
@@ -574,15 +574,22 @@ class CLI(object):
 		)
 
 	def _parse_inetnum_block(self, block):
-		logging.debug("Parsing inetnum block:")
+		log.debug("Parsing inetnum block:")
 
 		inetnum = {}
 		for line in block:
-			logging.debug(line)
+			log.debug(line)
 
 			# Split line
 			key, val = split_line(line)
 
+			# Filter any inetnum records which are only referring to IP space
+			# not managed by that specific RIR...
+			if key == "netname":
+				if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
+					log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
+					return
+
 			if key == "inetnum":
 				start_address, delim, end_address = val.partition("-")
 
@@ -594,7 +601,7 @@ class CLI(object):
 					start_address = ipaddress.ip_address(start_address)
 					end_address   = ipaddress.ip_address(end_address)
 				except ValueError:
-					logging.warning("Could not parse line: %s" % line)
+					log.warning("Could not parse line: %s" % line)
 					return
 
 				# Set prefix to default
@@ -611,15 +618,18 @@ class CLI(object):
 				inetnum[key] = val
 
 			elif key == "country":
-				if val == "UNITED STATES":
-					val = "US"
-
 				inetnum[key] = val.upper()
 
 		# Skip empty objects
 		if not inetnum or not "country" in inetnum:
 			return
 
+		# Skip objects with bogus country code 'ZZ'
+		if inetnum.get("country") == "ZZ":
+			log.warning("Skipping network with bogus country 'ZZ': %s" % \
+				(inetnum.get("inet6num") or inetnum.get("inetnum")))
+			return
+
 		network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
 
 		if not self._check_parsed_network(network):
-- 
2.20.1

From ebb087cfa30ec5ca0c96dcce66a91245c1ffc271 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Wed, 21 Oct 2020 14:47:43 +0000
Subject: [PATCH 026/111] location-importer.in: avoid log spam for too small
 networks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index a869256..864eab1 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -503,7 +503,7 @@ class CLI(object):
 				return False
 
 			if network.prefixlen > 24:
-				log.info("Skipping network too small to be publicly announced: %s" % network)
+				log.debug("Skipping network too small to be publicly announced: %s" % network)
 				return False
 
 			if str(network.network_address) == "0.0.0.0":
@@ -516,7 +516,7 @@ class CLI(object):
 				return False
 
 			if network.prefixlen > 48:
-				log.info("Skipping network too small to be publicly announced: %s" % network)
+				log.debug("Skipping network too small to be publicly announced: %s" % network)
 				return False
 
 			if str(network.network_address) == "::":
-- 
2.20.1

From bbed1fd2330e8efa6b413dc152a1a6ef2d771aac Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 27 Oct 2020 17:14:30 +0000
Subject: [PATCH 027/111] export: Flatten the tree before exporting it

This patch removes the possibility that any IP address ranges
might show up in multiple exported files.

If this was content from the database:

  * 10.0.0.0/16 - DE
  * 10.0.1.0/24 - FR

Then the IP address 10.0.1.1 would match for both countries.

The algorithm will now break the larger /16 subnet apart into
smaller subnets so that 10.0.1.0/24 is no longer overlapped.

There was some time spent on this to make this as efficient
as possible.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/export.py | 154 ++++++++++++++++++++++++++++++-------------
 1 file changed, 110 insertions(+), 44 deletions(-)

diff --git a/src/python/export.py b/src/python/export.py
index 5eaf43f..dd44332 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -89,28 +89,55 @@ class OutputWriter(object):
 
 	def write(self, network, subnets):
 		if self.flatten and self._flatten(network):
-			log.debug("Skipping writing network %s" % network)
+			log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
 			return
 
+		# Convert network into a Python object
+		_network = ipaddress.ip_network("%s" % network)
+
 		# Write the network when it has no subnets
 		if not subnets:
-			network = ipaddress.ip_network("%s" % network)
-			return self._write_network(network)
+			log.debug("Writing %s to %s" % (_network, self.f))
+			return self._write_network(_network)
+
+		# Convert subnets into Python objects
+		_subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
+
+		# Split the network into smaller bits so that
+		# we can accomodate for any gaps in it later
+		to_check = set()
+		for _subnet in _subnets:
+			to_check.update(
+				_network.address_exclude(_subnet)
+			)
+
+		# Clear the list of all subnets
+		subnets = []
+
+		# Check if all subnets to not overlap with anything else
+		while to_check:
+			subnet_to_check = to_check.pop()
 
-		# Collect all matching subnets
-		matching_subnets = []
+			for _subnet in _subnets:
+				# Drop this subnet if it equals one of the subnets
+				# or if it is subnet of one of them
+				if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
+					break
 
-		for subnet in sorted(subnets):
-			# Try to find the subnet in the database
-			n = self.db.lookup("%s" % subnet.network_address)
+				# Break it down if it overlaps
+				if subnet_to_check.overlaps(_subnet):
+					to_check.update(
+						subnet_to_check.address_exclude(_subnet)
+					)
+					break
 
-			# No entry or matching country means those IP addresses belong here
-			if not n or n.country_code == network.country_code:
-				matching_subnets.append(subnet)
+			# Add the subnet again as it passed the check
+			else:
+				subnets.append(subnet_to_check)
 
 		# Write all networks as compact as possible
-		for network in ipaddress.collapse_addresses(matching_subnets):
-			log.debug("Writing %s to database" % network)
+		for network in ipaddress.collapse_addresses(subnets):
+			log.debug("Writing %s to %s" % (network, self.f))
 			self._write_network(network)
 
 	def finish(self):
@@ -206,40 +233,40 @@ class Exporter(object):
 			# Get all networks that match the family
 			networks = self.db.search_networks(family=family)
 
-			# Materialise the generator into a list (uses quite some memory)
-			networks = list(networks)
+			# Create a stack with all networks in order where we can put items back
+			# again and retrieve them in the next iteration.
+			networks = BufferedStack(networks)
 
 			# Walk through all networks
-			for i, network in enumerate(networks):
-				_network = ipaddress.ip_network("%s" % network)
-
-				# Search for all subnets
-				subnets = set()
-
-				while i < len(networks):
-					subnet = networks[i+1]
-
-					if subnet.is_subnet_of(network):
-						_subnet = ipaddress.ip_network("%s" % subnet)
-
-						subnets.add(_subnet)
-						subnets.update(_network.address_exclude(_subnet))
-
-						i += 1
-					else:
+			for network in networks:
+				# Collect all networks which are a subnet of network
+				subnets = []
+				for subnet in networks:
+					# If the next subnet was not a subnet, we have to push
+					# it back on the stack and break this loop
+					if not subnet.is_subnet_of(network):
+						networks.push(subnet)
 						break
 
+					subnets.append(subnet)
+
 				# Write matching countries
-				try:
-					writers[network.country_code].write(network, subnets)
-				except KeyError:
-					pass
+				if network.country_code and network.country_code in writers:
+					# Mismatching subnets
+					gaps = [
+						subnet for subnet in subnets if not network.country_code == subnet.country_code
+					]
+
+					writers[network.country_code].write(network, gaps)
 
 				# Write matching ASNs
-				try:
-					writers[network.asn].write(network, subnets)
-				except KeyError:
-					pass
+				if network.asn and network.asn in writers:
+					# Mismatching subnets
+					gaps = [
+						subnet for subnet in subnets if not network.asn == subnet.asn
+					]
+
+					writers[network.asn].write(network, gaps)
 
 				# Handle flags
 				for flag in flags:
@@ -247,10 +274,19 @@ class Exporter(object):
 						# Fetch the "fake" country code
 						country = flags[flag]
 
-						try:
-							writers[country].write(network, subnets)
-						except KeyError:
-							pass
+						if not country in writers:
+							continue
+
+						gaps = [
+							subnet for subnet in subnets
+								if not subnet.has_flag(flag)
+						]
+
+						writers[country].write(network, gaps)
+
+				# Push all subnets back onto the stack
+				for subnet in reversed(subnets):
+					networks.push(subnet)
 
 			# Write everything to the filesystem
 			for writer in writers.values():
@@ -262,3 +298,33 @@ class Exporter(object):
 		)
 
 		return os.path.join(directory, filename)
+
+
+class BufferedStack(object):
+	"""
+		This class takes an iterator and when being iterated
+		over it returns objects from that iterator for as long
+		as there are any.
+
+		It additionally has a function to put an item back on
+		the back so that it will be returned again at the next
+		iteration.
+	"""
+	def __init__(self, iterator):
+		self.iterator = iterator
+		self.stack = []
+
+	def __iter__(self):
+		return self
+
+	def __next__(self):
+		if self.stack:
+			return self.stack.pop(0)
+
+		return next(self.iterator)
+
+	def push(self, elem):
+		"""
+			Takes an element and puts it on the stack
+		"""
+		self.stack.insert(0, elem)
-- 
2.20.1

From e99a72265c1ba2194b61663eda7e9f14e0083016 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 28 Oct 2020 09:52:36 +0000
Subject: [PATCH 028/111] location: Fix Python syntax error in verify()

The database is now being opened before the requested
method is called and handle_verify() wasn't updated.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location.in | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/python/location.in b/src/python/location.in
index 44ad726..b5e5758 100644
--- a/src/python/location.in
+++ b/src/python/location.in
@@ -453,13 +453,7 @@ class CLI(object):
 
 		return 0
 
-	def handle_verify(self, ns):
-		try:
-			db = location.Database(ns.database)
-		except FileNotFoundError as e:
-			log.error("%s: %s" % (ns.database, e))
-			return 127
-
+	def handle_verify(self, db, ns):
 		# Verify the database
 		with open(ns.public_key, "r") as f:
 			if not db.verify(f):
-- 
2.20.1

From 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Thu, 29 Oct 2020 07:25:53 -0700
Subject: [PATCH 029/111] location update: Remove double conversion of
 timestamps
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This caused that the timestamp in DNS was misinterpreted
as local time and often, databases could not be downloaded.

Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
---
 src/python/location.in | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/python/location.in b/src/python/location.in
index b5e5758..070640c 100644
--- a/src/python/location.in
+++ b/src/python/location.in
@@ -421,11 +421,8 @@ class CLI(object):
 		# Fetch the timestamp we need from DNS
 		t = location.discover_latest_version()
 
-		# Parse timestamp into datetime format
-		timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
-
 		# Check the version of the local database
-		if db and timestamp and db.created_at >= timestamp.timestamp():
+		if db and t and db.created_at >= t:
 			log.info("Already on the latest version")
 			return
 
-- 
2.20.1

From 60c1ac0307312614bd6980d30b44bb59b5a6ab6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Thu, 29 Oct 2020 07:36:46 -0700
Subject: [PATCH 030/111] location.in: do not confuse UTC with local time zones
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
---
 src/python/location.in | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/python/location.in b/src/python/location.in
index 070640c..0d09210 100644
--- a/src/python/location.in
+++ b/src/python/location.in
@@ -398,10 +398,7 @@ class CLI(object):
 
 	def handle_update(self, db, ns):
 		if ns.cron and db:
-			now = datetime.datetime.utcnow()
-
-			# Parse the database timestamp
-			t = datetime.datetime.utcfromtimestamp(db.created_at)
+			now = time.time()
 
 			if ns.cron == "daily":
 				delta = datetime.timedelta(days=1)
@@ -410,11 +407,12 @@ class CLI(object):
 			elif ns.cron == "monthly":
 				delta = datetime.timedelta(days=30)
 
+			delta = delta.total_seconds()
+
 			# Check if the database has recently been updated
-			if t >= (now - delta):
+			if db.created_at >= (now - delta):
 				log.info(
-					_("The database has been updated recently (%s)") % \
-						format_timedelta(now - t),
+					_("The database has been updated recently"),
 				)
 				return 3
 
-- 
2.20.1

From e7d612e5219ef9ba612ed404e4e2c174110d3dd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
Date: Tue, 3 Nov 2020 15:31:08 +0000
Subject: [PATCH 031/111] location-importer.in: always convert organisation
 handles into upper cases
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes: #12523

Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location-importer.in | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index 864eab1..2dec89e 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -560,7 +560,7 @@ class CLI(object):
 					autnum["asn"] = m.group(2)
 
 			elif key == "org":
-				autnum[key] = val
+				autnum[key] = val.upper()
 
 		# Skip empty objects
 		if not autnum:
@@ -646,7 +646,9 @@ class CLI(object):
 			# Split line
 			key, val = split_line(line)
 
-			if key in ("organisation", "org-name"):
+			if key == "organisation":
+				org[key] = val.upper()
+			elif key == "org-name":
 				org[key] = val
 
 		# Skip empty objects
-- 
2.20.1

From e96704f43acca1a8f56d9a680cce281f5e587ec5 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 11 Nov 2020 21:16:45 +0000
Subject: [PATCH 032/111] test: Add tests for database enumerator

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/test-database.c | 53 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/src/test-database.c b/src/test-database.c
index b4a75c4..4aef94e 100644
--- a/src/test-database.c
+++ b/src/test-database.c
@@ -38,6 +38,14 @@ const char* DESCRIPTION =
 	"Maecenas ut venenatis nunc.";
 const char* LICENSE = "CC";
 
+const char* networks[] = {
+	"2001:db8::/32",
+	"2001:db8:1000::/48",
+	"2001:db8:2000::/48",
+	"2001:db8:2020::/48",
+	NULL,
+};
+
 static int attempt_to_open(struct loc_ctx* ctx, char* path) {
 	FILE* f = fopen(path, "r");
 	if (!f)
@@ -139,6 +147,24 @@ int main(int argc, char** argv) {
 		exit(EXIT_FAILURE);
 	}
 
+	struct loc_network* network = NULL;
+
+	// Add some networks
+	const char** n = networks;
+	while (*n) {
+		err = loc_writer_add_network(writer, &network, *n);
+		if (err) {
+			fprintf(stderr, "Could not add network %s\n", *n);
+			exit(EXIT_FAILURE);
+		}
+
+		// Set a country
+		loc_network_set_country_code(network, "XX");
+
+		// Next one
+		n++;
+	}
+
 	FILE* f = tmpfile();
 	if (!f) {
 		fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
@@ -170,6 +196,33 @@ int main(int argc, char** argv) {
 		exit(EXIT_FAILURE);
 	}
 
+	// Enumerator
+	struct loc_database_enumerator* enumerator;
+	err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
+	if (err) {
+		fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
+		exit(EXIT_FAILURE);
+	}
+
+	// Walk through all networks
+	while (1) {
+		err = loc_database_enumerator_next_network(enumerator, &network);
+		if (err) {
+			fprintf(stderr, "Error fetching the next network: %d\n", err);
+			exit(EXIT_FAILURE);
+		}
+
+		if (!network)
+			break;
+
+		char* s = loc_network_str(network);
+		printf("Got network: %s\n", s);
+		free(s);
+	}
+
+	// Free the enumerator
+	loc_database_enumerator_unref(enumerator);
+
 	// Close the database
 	loc_database_unref(db);
 	loc_unref(ctx);
-- 
2.20.1

From ecce288da39a2c0eb60076050ca21e9619f61844 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 11 Nov 2020 23:01:19 +0000
Subject: [PATCH 033/111] networks: Add list to manage groups of networks

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym    | 11 ++++++
 src/loc/network.h | 12 ++++++
 src/network.c     | 96 +++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 119 insertions(+)

diff --git a/src/libloc.sym b/src/libloc.sym
index b8296eb..6ef2d27 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -98,6 +98,17 @@ global:
 	loc_network_str;
 	loc_network_unref;
 
+	# Network List
+	loc_network_list_clear;
+	loc_network_list_empty;
+	loc_network_list_get;
+	loc_network_list_new;
+	loc_network_list_pop;
+	loc_network_list_push;
+	loc_network_list_ref;
+	loc_network_list_size;
+	loc_network_list_unref;
+
 	# Writer
 	loc_writer_add_as;
 	loc_writer_add_country;
diff --git a/src/loc/network.h b/src/loc/network.h
index 70c3803..fd20812 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -56,6 +56,18 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
 
 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
 
+// List
+struct loc_network_list;
+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
+size_t loc_network_list_size(struct loc_network_list* list);
+int loc_network_list_empty(struct loc_network_list* list);
+void loc_network_list_clear(struct loc_network_list* list);
+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
+struct loc_network* loc_network_list_pop(struct loc_network_list* list);
+
 #ifdef LIBLOC_PRIVATE
 
 int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
diff --git a/src/network.c b/src/network.c
index d7b1645..c9e7979 100644
--- a/src/network.c
+++ b/src/network.c
@@ -746,3 +746,99 @@ LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node)
 LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
 	return loc_network_ref(node->network);
 }
+
+// List
+
+struct loc_network_list {
+	struct loc_ctx* ctx;
+	int refcount;
+
+	struct loc_network* list[1024];
+	size_t size;
+	size_t max_size;
+};
+
+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
+		struct loc_network_list** list) {
+	struct loc_network_list* l = calloc(1, sizeof(*l));
+	if (!l)
+		return -ENOMEM;
+
+	l->ctx = loc_ref(ctx);
+	l->refcount = 1;
+
+	// Do not allow this list to grow larger than this
+	l->max_size = 1024;
+
+	DEBUG(l->ctx, "Network list allocated at %p\n", l);
+	*list = l;
+	return 0;
+}
+
+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
+	list->refcount++;
+
+	return list;
+}
+
+static void loc_network_list_free(struct loc_network_list* list) {
+	DEBUG(list->ctx, "Releasing network list at %p\n", list);
+
+	for (unsigned int i = 0; i < list->size; i++)
+		loc_network_unref(list->list[i]);
+
+	loc_unref(list->ctx);
+	free(list);
+}
+
+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
+	if (!list)
+		return NULL;
+
+	if (--list->refcount > 0)
+		return list;
+
+	loc_network_list_free(list);
+	return NULL;
+}
+
+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
+	return list->size;
+}
+
+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
+	return list->size == 0;
+}
+
+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
+	for (unsigned int i = 0; i < list->size; i++)
+		loc_network_unref(list->list[i]);
+
+	list->size = 0;
+}
+
+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
+	// Check index
+	if (index >= list->size)
+		return NULL;
+
+	return loc_network_ref(list->list[index]);
+}
+
+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
+	// Check if we have space left
+	if (list->size == list->max_size)
+		return -ENOMEM;
+
+	list->list[list->size++] = loc_network_ref(network);
+
+	return 0;
+}
+
+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
+	// Return nothing when empty
+	if (loc_network_list_empty(list))
+		return NULL;
+
+	return list->list[list->size--];
+}
-- 
2.20.1

From 8b2205272b7872a1458ad87811abf58609f38ad4 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 13:57:35 +0000
Subject: [PATCH 034/111] networks: Add function to dump lists

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym    |  1 +
 src/loc/network.h |  1 +
 src/network.c     | 14 ++++++++++++++
 3 files changed, 16 insertions(+)

diff --git a/src/libloc.sym b/src/libloc.sym
index 6ef2d27..a5641c6 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -100,6 +100,7 @@ global:
 
 	# Network List
 	loc_network_list_clear;
+	loc_network_list_dump;
 	loc_network_list_empty;
 	loc_network_list_get;
 	loc_network_list_new;
diff --git a/src/loc/network.h b/src/loc/network.h
index fd20812..44c50a4 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -64,6 +64,7 @@ struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
 size_t loc_network_list_size(struct loc_network_list* list);
 int loc_network_list_empty(struct loc_network_list* list);
 void loc_network_list_clear(struct loc_network_list* list);
+void loc_network_list_dump(struct loc_network_list* list);
 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
diff --git a/src/network.c b/src/network.c
index c9e7979..0977406 100644
--- a/src/network.c
+++ b/src/network.c
@@ -817,6 +817,20 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
 	list->size = 0;
 }
 
+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
+	struct loc_network* network;
+	char* s;
+
+	for (unsigned int i = 0; i < list->size; i++) {
+		network = list->list[i];
+
+		s = loc_network_str(network);
+
+		INFO(list->ctx, "%s\n", s);
+		free(s);
+	}
+}
+
 LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
 	// Check index
 	if (index >= list->size)
-- 
2.20.1

From 850e75167e8e03fe8b951992c9f7bc2ccb9fb711 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 14:18:40 +0000
Subject: [PATCH 035/111] network: Add functions to break network into subnets

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym     |   5 +
 src/loc/network.h  |   6 +
 src/network.c      | 290 ++++++++++++++++++++++++++++++++++++++++++++-
 src/test-network.c |  61 ++++++++++
 4 files changed, 361 insertions(+), 1 deletion(-)

diff --git a/src/libloc.sym b/src/libloc.sym
index a5641c6..fcb9ea5 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -80,6 +80,8 @@ global:
 
 	# Network
 	loc_network_address_family;
+	loc_network_eq;
+	loc_network_exclude;
 	loc_network_format_first_address;
 	loc_network_format_last_address;
 	loc_network_get_asn;
@@ -96,6 +98,7 @@ global:
 	loc_network_set_country_code;
 	loc_network_set_flag;
 	loc_network_str;
+	loc_network_subnets;
 	loc_network_unref;
 
 	# Network List
@@ -107,7 +110,9 @@ global:
 	loc_network_list_pop;
 	loc_network_list_push;
 	loc_network_list_ref;
+	loc_network_list_reverse;
 	loc_network_list_size;
+	loc_network_list_sort;
 	loc_network_list_unref;
 
 	# Writer
diff --git a/src/loc/network.h b/src/loc/network.h
index 44c50a4..4e51a62 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -54,7 +54,11 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
 
+int loc_network_eq(struct loc_network* self, struct loc_network* other);
 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
+struct loc_network_list* loc_network_subnets(struct loc_network* network);
+struct loc_network_list* loc_network_exclude(
+		struct loc_network* self, struct loc_network* other);
 
 // List
 struct loc_network_list;
@@ -68,6 +72,8 @@ void loc_network_list_dump(struct loc_network_list* list);
 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
+void loc_network_list_sort(struct loc_network_list* list);
+void loc_network_list_reverse(struct loc_network_list* list);
 
 #ifdef LIBLOC_PRIVATE
 
diff --git a/src/network.c b/src/network.c
index 0977406..6c08070 100644
--- a/src/network.c
+++ b/src/network.c
@@ -97,6 +97,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
 	return a;
 }
 
+static struct in6_addr address_increment(const struct in6_addr* address) {
+	struct in6_addr a = *address;
+
+	for (int octet = 15; octet >= 0; octet--) {
+		if (a.s6_addr[octet] < 255) {
+			a.s6_addr[octet]++;
+			break;
+		} else {
+			a.s6_addr[octet] = 0;
+		}
+	}
+
+	return a;
+}
+
 LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
 		struct in6_addr* address, unsigned int prefix) {
 	// Address cannot be unspecified
@@ -405,6 +420,69 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
 	return loc_network_has_flag(network, flag);
 }
 
+LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
+#ifdef ENABLE_DEBUG
+	char* n1 = loc_network_str(self);
+	char* n2 = loc_network_str(other);
+
+	DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
+
+	free(n1);
+	free(n2);
+#endif
+
+	// Family must be the same
+	if (self->family != other->family) {
+		DEBUG(self->ctx, "  Family mismatch\n");
+
+		return 0;
+	}
+
+	// The start address must be the same
+	if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
+		DEBUG(self->ctx, "  Address mismatch\n");
+
+		return 0;
+	}
+
+	// The prefix length must be the same
+	if (self->prefix != other->prefix) {
+		DEBUG(self->ctx, "  Prefix mismatch\n");
+		return 0;
+	}
+
+	DEBUG(self->ctx, "  Yes!\n");
+
+	return 1;
+}
+
+static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
+	// Families must match
+	if (self->family != other->family)
+		return -1;
+
+	int r = in6_addr_cmp(&self->first_address, &other->first_address);
+
+	switch (r) {
+		// Smaller
+		case -1:
+			return 0;
+
+		// Larger
+		case 1:
+			return 1;
+
+		default:
+			break;
+	}
+
+	if (self->prefix > other->prefix)
+		return 1;
+
+	// Dunno
+	return 0;
+}
+
 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
 	// If the start address of the other network is smaller than this network,
 	// it cannot be a subnet.
@@ -419,6 +497,175 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net
 	return 1;
 }
 
+LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
+	struct loc_network_list* list;
+
+	// New prefix length
+	unsigned int prefix = network->prefix + 1;
+
+	// Check if the new prefix is valid
+	if (valid_prefix(&network->first_address, prefix))
+		return NULL;
+
+	// Create a new list with the result
+	int r = loc_network_list_new(network->ctx, &list);
+	if (r) {
+		ERROR(network->ctx, "Could not create network list: %d\n", r);
+		return NULL;
+	}
+
+	struct loc_network* subnet1 = NULL;
+	struct loc_network* subnet2 = NULL;
+
+	// Create the first half of the network
+	r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix);
+	if (r)
+		goto ERROR;
+
+	// The next subnet starts after the first one
+	struct in6_addr first_address = address_increment(&subnet1->last_address);
+
+	// Create the second half of the network
+	r = loc_network_new(network->ctx, &subnet2, &first_address, prefix);
+	if (r)
+		goto ERROR;
+
+	// Push the both onto the stack (in reverse order)
+	r = loc_network_list_push(list, subnet2);
+	if (r)
+		goto ERROR;
+
+	r = loc_network_list_push(list, subnet1);
+	if (r)
+		goto ERROR;
+
+	loc_network_unref(subnet1);
+	loc_network_unref(subnet2);
+
+	return list;
+
+ERROR:
+	if (subnet1)
+		loc_network_unref(subnet1);
+
+	if (subnet2)
+		loc_network_unref(subnet2);
+
+	if (list)
+		loc_network_list_unref(list);
+
+	return NULL;
+}
+
+LOC_EXPORT struct loc_network_list* loc_network_exclude(
+		struct loc_network* self, struct loc_network* other) {
+	struct loc_network_list* list;
+
+#ifdef ENABLE_DEBUG
+	char* n1 = loc_network_str(self);
+	char* n2 = loc_network_str(other);
+
+	DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
+
+	free(n1);
+	free(n2);
+#endif
+
+	// Family must match
+	if (self->family != other->family) {
+		DEBUG(self->ctx, "Family mismatch\n");
+
+		return NULL;
+	}
+
+	// Other must be a subnet of self
+	if (!loc_network_is_subnet_of(other, self)) {
+		DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
+
+		return NULL;
+	}
+
+	// We cannot perform this operation if both networks equal
+	if (loc_network_eq(self, other)) {
+		DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
+
+		return NULL;
+	}
+
+	// Create a new list with the result
+	int r = loc_network_list_new(self->ctx, &list);
+	if (r) {
+		ERROR(self->ctx, "Could not create network list: %d\n", r);
+		return NULL;
+	}
+
+	struct loc_network_list* subnets = loc_network_subnets(self);
+
+	struct loc_network* subnet1 = NULL;
+	struct loc_network* subnet2 = NULL;
+
+	while (subnets) {
+		// Fetch both subnets
+		subnet1 = loc_network_list_get(subnets, 0);
+		subnet2 = loc_network_list_get(subnets, 1);
+
+		// Free list
+		loc_network_list_unref(subnets);
+		subnets = NULL;
+
+		if (loc_network_eq(other, subnet1)) {
+			r = loc_network_list_push(list, subnet2);
+			if (r)
+				goto ERROR;
+
+		} else if (loc_network_eq(other, subnet2)) {
+			r = loc_network_list_push(list, subnet1);
+			if (r)
+				goto ERROR;
+
+		} else  if (loc_network_is_subnet_of(other, subnet1)) {
+			r = loc_network_list_push(list, subnet2);
+			if (r)
+				goto ERROR;
+
+			subnets = loc_network_subnets(subnet1);
+
+		} else if (loc_network_is_subnet_of(other, subnet2)) {
+			r = loc_network_list_push(list, subnet1);
+			if (r)
+				goto ERROR;
+
+			subnets = loc_network_subnets(subnet2);
+
+		} else {
+			ERROR(self->ctx, "We should never get here\n");
+			goto ERROR;
+		}
+
+		loc_network_unref(subnet1);
+		loc_network_unref(subnet2);
+	}
+
+#ifdef ENABLE_DEBUG
+	loc_network_list_dump(list);
+#endif
+
+	// Return the result
+	return list;
+
+ERROR:
+	if (subnet1)
+		loc_network_unref(subnet1);
+
+	if (subnet2)
+		loc_network_unref(subnet2);
+
+	if (list)
+		loc_network_list_unref(list);
+
+	return NULL;
+}
+
 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
 	// Add country code
 	loc_country_code_copy(dbobj->country_code, network->country_code);
@@ -854,5 +1101,46 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
 	if (loc_network_list_empty(list))
 		return NULL;
 
-	return list->list[list->size--];
+	return list->list[--list->size];
+}
+
+static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
+	// Do nothing for invalid indices
+	if (i1 >= list->size || i2 >= list->size)
+		return;
+
+	DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
+
+	struct loc_network* network1 = list->list[i1];
+	struct loc_network* network2 = list->list[i2];
+
+	list->list[i1] = network2;
+	list->list[i2] = network1;
+}
+
+LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
+	unsigned int i = 0;
+	unsigned int j = list->size - 1;
+
+	while (i < j) {
+		loc_network_list_swap(list, i++, j--);
+	}
+}
+
+LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
+	unsigned int n = list->size;
+	int swapped;
+
+	do {
+		swapped = 0;
+
+		for (unsigned int i = 1; i < n; i++) {
+			if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
+				loc_network_list_swap(list, i-1, i);
+				swapped = 1;
+			}
+		}
+
+		n--;
+	} while (swapped);
 }
diff --git a/src/test-network.c b/src/test-network.c
index b6776b4..af1b2e6 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -118,6 +118,19 @@ int main(int argc, char** argv) {
 	size_t nodes = loc_network_tree_count_nodes(tree);
 	printf("The tree has %zu nodes\n", nodes);
 
+	// Check equals function
+	err = loc_network_eq(network1, network1);
+	if (!err) {
+		fprintf(stderr, "Network is not equal with itself\n");
+		exit(EXIT_FAILURE);
+	}
+
+	err = loc_network_eq(network1, network2);
+	if (err) {
+		fprintf(stderr, "Networks equal unexpectedly\n");
+		exit(EXIT_FAILURE);
+	}
+
 	// Check subnet function
 	err = loc_network_is_subnet_of(network1, network2);
 	if (err != 0) {
@@ -131,6 +144,54 @@ int main(int argc, char** argv) {
 		exit(EXIT_FAILURE);
 	}
 
+	// Make list of subnets
+	struct loc_network_list* subnets = loc_network_subnets(network1);
+	if (!subnets) {
+		fprintf(stderr, "Could not find subnets of network\n");
+		exit(EXIT_FAILURE);
+	}
+
+	loc_network_list_dump(subnets);
+
+	while (!loc_network_list_empty(subnets)) {
+		struct loc_network* subnet = loc_network_list_pop(subnets);
+		if (!subnet) {
+			fprintf(stderr, "Received an empty subnet\n");
+			exit(EXIT_FAILURE);
+		}
+
+		char* s = loc_network_str(subnet);
+		printf("Received subnet %s\n", s);
+		free(s);
+
+		if (!loc_network_is_subnet_of(subnet, network1)) {
+			fprintf(stderr, "Not a subnet\n");
+			exit(EXIT_FAILURE);
+		}
+
+		loc_network_unref(subnet);
+	}
+
+	loc_network_list_unref(subnets);
+
+	struct loc_network_list* excluded = loc_network_exclude(network1, network2);
+	if (!excluded) {
+		fprintf(stderr, "Could not create excluded list\n");
+		exit(EXIT_FAILURE);
+	}
+
+	loc_network_list_dump(excluded);
+
+	// Reverse it
+	loc_network_list_reverse(excluded);
+	loc_network_list_dump(excluded);
+
+	// Sort them and dump them again
+	loc_network_list_sort(excluded);
+	loc_network_list_dump(excluded);
+
+	loc_network_list_unref(excluded);
+
 	// Create a database
 	struct loc_writer* writer;
 	err = loc_writer_new(ctx, &writer, NULL, NULL);
-- 
2.20.1

From 6159d384c4a98fe45ec52522e2950719e4982d80 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 14:24:58 +0000
Subject: [PATCH 036/111] networks: Add function to check if two networks
 overlap

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym    |  1 +
 src/loc/network.h |  1 +
 src/network.c     | 16 ++++++++++++++++
 3 files changed, 18 insertions(+)

diff --git a/src/libloc.sym b/src/libloc.sym
index fcb9ea5..0c2835b 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -93,6 +93,7 @@ global:
 	loc_network_match_flag;
 	loc_network_new;
 	loc_network_new_from_string;
+	loc_network_overlaps;
 	loc_network_ref;
 	loc_network_set_asn;
 	loc_network_set_country_code;
diff --git a/src/loc/network.h b/src/loc/network.h
index 4e51a62..ef13756 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -55,6 +55,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
 
 int loc_network_eq(struct loc_network* self, struct loc_network* other);
+int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
 struct loc_network_list* loc_network_subnets(struct loc_network* network);
 struct loc_network_list* loc_network_exclude(
diff --git a/src/network.c b/src/network.c
index 6c08070..d826511 100644
--- a/src/network.c
+++ b/src/network.c
@@ -483,6 +483,22 @@ static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
 	return 0;
 }
 
+LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
+	if (loc_network_match_address(self, &other->first_address) == 0)
+		return 1;
+
+	if (loc_network_match_address(self, &other->last_address) == 0)
+		return 1;
+
+	if (loc_network_match_address(other, &self->first_address) == 0)
+		return 1;
+
+	if (loc_network_match_address(other, &self->last_address) == 0)
+		return 1;
+
+	return 0;
+}
+
 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
 	// If the start address of the other network is smaller than this network,
 	// it cannot be a subnet.
-- 
2.20.1

From e52ba21761f27e592040a2793b2a26bbeeeecc05 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 14:28:15 +0000
Subject: [PATCH 037/111] networks: Add function to check if network is part of
 a list

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym    | 1 +
 src/loc/network.h | 1 +
 src/network.c     | 9 +++++++++
 3 files changed, 11 insertions(+)

diff --git a/src/libloc.sym b/src/libloc.sym
index 0c2835b..f1b63a2 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -104,6 +104,7 @@ global:
 
 	# Network List
 	loc_network_list_clear;
+	loc_network_list_contains;
 	loc_network_list_dump;
 	loc_network_list_empty;
 	loc_network_list_get;
diff --git a/src/loc/network.h b/src/loc/network.h
index ef13756..7804512 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -73,6 +73,7 @@ void loc_network_list_dump(struct loc_network_list* list);
 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
 void loc_network_list_sort(struct loc_network_list* list);
 void loc_network_list_reverse(struct loc_network_list* list);
 
diff --git a/src/network.c b/src/network.c
index d826511..fcbdc59 100644
--- a/src/network.c
+++ b/src/network.c
@@ -1120,6 +1120,15 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
 	return list->list[--list->size];
 }
 
+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
+	for (unsigned int i = 0; i < list->size; i++) {
+		if (loc_network_eq(list->list[i], network))
+			return 1;
+	}
+
+	return 0;
+}
+
 static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
 	// Do nothing for invalid indices
 	if (i1 >= list->size || i2 >= list->size)
-- 
2.20.1

From f802f3a4decf4827ecc8bcabe269ae9f94f7f32d Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 14:33:22 +0000
Subject: [PATCH 038/111] networks: Add function to merge two lists

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym    |  1 +
 src/loc/network.h |  1 +
 src/network.c     | 13 +++++++++++++
 3 files changed, 15 insertions(+)

diff --git a/src/libloc.sym b/src/libloc.sym
index f1b63a2..c0b6b1f 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -108,6 +108,7 @@ global:
 	loc_network_list_dump;
 	loc_network_list_empty;
 	loc_network_list_get;
+	loc_network_list_merge;
 	loc_network_list_new;
 	loc_network_list_pop;
 	loc_network_list_push;
diff --git a/src/loc/network.h b/src/loc/network.h
index 7804512..e30d91c 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -76,6 +76,7 @@ struct loc_network* loc_network_list_pop(struct loc_network_list* list);
 int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
 void loc_network_list_sort(struct loc_network_list* list);
 void loc_network_list_reverse(struct loc_network_list* list);
+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
 
 #ifdef LIBLOC_PRIVATE
 
diff --git a/src/network.c b/src/network.c
index fcbdc59..541286d 100644
--- a/src/network.c
+++ b/src/network.c
@@ -1169,3 +1169,16 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
 		n--;
 	} while (swapped);
 }
+
+LOC_EXPORT int loc_network_list_merge(
+		struct loc_network_list* self, struct loc_network_list* other) {
+	int r;
+
+	for (unsigned int i = 0; i < other->size; i++) {
+		r = loc_network_list_push(self, other->list[i]);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
-- 
2.20.1

From 6d22a179dffd08fcf2a44aafb361725ab22486d4 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 14:35:43 +0000
Subject: [PATCH 039/111] network: Make lists unique

Networks that are in the list won't be added again

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/network.c b/src/network.c
index 541286d..44571b3 100644
--- a/src/network.c
+++ b/src/network.c
@@ -1103,6 +1103,10 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
 }
 
 LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
+	// Do not add networks that are already on the list
+	if (loc_network_list_contains(list, network))
+		return 0;
+
 	// Check if we have space left
 	if (list->size == list->max_size)
 		return -ENOMEM;
-- 
2.20.1

From 681ff05cb7cdf230d38abf09a330a31498e265a4 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 19:21:13 +0000
Subject: [PATCH 040/111] database: Pass flag to enumerator to flatten output

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c        |  8 +++++++-
 src/loc/database.h    |  6 +++++-
 src/perl/Location.xs  |  2 +-
 src/python/database.c | 25 ++++++++++++++++++-------
 4 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/src/database.c b/src/database.c
index fa1dad0..9baab33 100644
--- a/src/database.c
+++ b/src/database.c
@@ -104,6 +104,9 @@ struct loc_database_enumerator {
 	enum loc_network_flags flags;
 	int family;
 
+	// Flatten output?
+	int flatten;
+
 	// Index of the AS we are looking at
 	unsigned int as_index;
 
@@ -933,7 +936,7 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
 // Enumerator
 
 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
-		struct loc_database* db, enum loc_database_enumerator_mode mode) {
+		struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
 	struct loc_database_enumerator* e = calloc(1, sizeof(*e));
 	if (!e)
 		return -ENOMEM;
@@ -944,6 +947,9 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
 	e->mode = mode;
 	e->refcount = 1;
 
+	// Flatten output?
+	e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
+
 	// Initialise graph search
 	//e->network_stack[++e->network_stack_depth] = 0;
 	e->network_stack_depth = 1;
diff --git a/src/loc/database.h b/src/loc/database.h
index 43173dd..14eb5ea 100644
--- a/src/loc/database.h
+++ b/src/loc/database.h
@@ -55,9 +55,13 @@ enum loc_database_enumerator_mode {
 	LOC_DB_ENUMERATE_COUNTRIES = 3,
 };
 
+enum loc_database_enumerator_flags {
+	LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
+};
+
 struct loc_database_enumerator;
 int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
-	struct loc_database* db, enum loc_database_enumerator_mode mode);
+	struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
 struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
 
diff --git a/src/perl/Location.xs b/src/perl/Location.xs
index dcf3f0d..b7676d2 100644
--- a/src/perl/Location.xs
+++ b/src/perl/Location.xs
@@ -125,7 +125,7 @@ database_countries(db)
 	PPCODE:
 		// Create Database enumerator
 		struct loc_database_enumerator* enumerator;
-		int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
+		int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
 
 		if (err) {
 			croak("Could not create a database enumerator\n");
diff --git a/src/python/database.c b/src/python/database.c
index 1013a58..7f8c2c2 100644
--- a/src/python/database.c
+++ b/src/python/database.c
@@ -207,10 +207,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
 	return (PyObject*)self;
 }
 
-static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
+static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
 	struct loc_database_enumerator* enumerator;
 
-	int r = loc_database_enumerator_new(&enumerator, self->db, what);
+	int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
 	if (r) {
 		PyErr_SetFromErrno(PyExc_SystemError);
 		return NULL;
@@ -223,7 +223,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
 }
 
 static PyObject* Database_ases(DatabaseObject* self) {
-	return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
+	return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
 }
 
 static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
@@ -234,7 +234,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
 
 	struct loc_database_enumerator* enumerator;
 
-	int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
+	int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
 	if (r) {
 		PyErr_SetFromErrno(PyExc_SystemError);
 		return NULL;
@@ -250,7 +250,11 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
 }
 
 static PyObject* Database_networks(DatabaseObject* self) {
-	return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
+	return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
+}
+
+static PyObject* Database_networks_flattened(DatabaseObject *self) {
+	return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
 }
 
 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
@@ -264,7 +268,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
 		return NULL;
 
 	struct loc_database_enumerator* enumerator;
-	int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
+	int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
 	if (r) {
 		PyErr_SetFromErrno(PyExc_SystemError);
 		return NULL;
@@ -317,7 +321,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
 }
 
 static PyObject* Database_countries(DatabaseObject* self) {
-	return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
+	return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
 }
 
 static struct PyMethodDef Database_methods[] = {
@@ -403,6 +407,13 @@ static struct PyGetSetDef Database_getsetters[] = {
 		NULL,
 		NULL,
 	},
+	{
+		"networks_flattened",
+		(getter)Database_networks_flattened,
+		NULL,
+		NULL,
+		NULL,
+	},
 	{
 		"vendor",
 		(getter)Database_get_vendor,
-- 
2.20.1

From f5e50a47e37e9b29d0d2ee9e5a41e5a5fe5aea7f Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 19:21:58 +0000
Subject: [PATCH 041/111] network: Reduce debugging output

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 28 +++-------------------------
 1 file changed, 3 insertions(+), 25 deletions(-)

diff --git a/src/network.c b/src/network.c
index 44571b3..f7071a6 100644
--- a/src/network.c
+++ b/src/network.c
@@ -421,37 +421,17 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
 }
 
 LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
-#ifdef ENABLE_DEBUG
-	char* n1 = loc_network_str(self);
-	char* n2 = loc_network_str(other);
-
-	DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
-
-	free(n1);
-	free(n2);
-#endif
-
 	// Family must be the same
-	if (self->family != other->family) {
-		DEBUG(self->ctx, "  Family mismatch\n");
-
+	if (self->family != other->family)
 		return 0;
-	}
 
 	// The start address must be the same
-	if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
-		DEBUG(self->ctx, "  Address mismatch\n");
-
+	if (in6_addr_cmp(&self->first_address, &other->first_address) != 0)
 		return 0;
-	}
 
 	// The prefix length must be the same
-	if (self->prefix != other->prefix) {
-		DEBUG(self->ctx, "  Prefix mismatch\n");
+	if (self->prefix != other->prefix)
 		return 0;
-	}
-
-	DEBUG(self->ctx, "  Yes!\n");
 
 	return 1;
 }
@@ -1138,8 +1118,6 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
 	if (i1 >= list->size || i2 >= list->size)
 		return;
 
-	DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
-
 	struct loc_network* network1 = list->list[i1];
 	struct loc_network* network2 = list->list[i2];
 
-- 
2.20.1

From 037c65d3a07ec6d37ff063f0645adda6b483b407 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 19:36:38 +0000
Subject: [PATCH 042/111] python: Export networks exclude function

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/network.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/src/python/network.c b/src/python/network.c
index 5496d1e..11f672b 100644
--- a/src/python/network.c
+++ b/src/python/network.c
@@ -24,6 +24,24 @@
 #include "locationmodule.h"
 #include "network.h"
 
+static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
+	PyObject* list = PyList_New(0);
+	if (!networks)
+		return list;
+
+	while (!loc_network_list_empty(networks)) {
+		struct loc_network* network = loc_network_list_pop(networks);
+
+		PyObject* n = new_network(&NetworkType, network);
+		PyList_Append(list, n);
+
+		loc_network_unref(network);
+		Py_DECREF(n);
+	}
+
+	return list;
+}
+
 PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
 	NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
 	if (self) {
@@ -154,6 +172,21 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
 	Py_RETURN_NONE;
 }
 
+static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
+	NetworkObject* other = NULL;
+
+	if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
+		return NULL;
+
+	struct loc_network_list* list = loc_network_exclude(self->network, other->network);
+
+	// Convert to Python objects
+	PyObject* obj = PyList_FromNetworkList(list);
+	loc_network_list_unref(list);
+
+	return obj;
+}
+
 static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
 	NetworkObject* other = NULL;
 
@@ -191,6 +224,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
 }
 
 static struct PyMethodDef Network_methods[] = {
+	{
+		"exclude",
+		(PyCFunction)Network_exclude,
+		METH_VARARGS,
+		NULL,
+	},
 	{
 		"has_flag",
 		(PyCFunction)Network_has_flag,
-- 
2.20.1

From 9a7732c8679e805d4d2d55ea4750c5d70ca4bd2c Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 19:59:22 +0000
Subject: [PATCH 043/111] network: Add more debugging output to stacks

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/network.c b/src/network.c
index f7071a6..d41e873 100644
--- a/src/network.c
+++ b/src/network.c
@@ -1088,8 +1088,12 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
 		return 0;
 
 	// Check if we have space left
-	if (list->size == list->max_size)
+	if (list->size == list->max_size) {
+		ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
 		return -ENOMEM;
+	}
+
+	DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
 
 	list->list[list->size++] = loc_network_ref(network);
 
@@ -1098,10 +1102,16 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
 
 LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
 	// Return nothing when empty
-	if (loc_network_list_empty(list))
+	if (loc_network_list_empty(list)) {
+		DEBUG(list->ctx, "%p: Popped empty stack\n", list);
 		return NULL;
+	}
 
-	return list->list[--list->size];
+	struct loc_network* network = list->list[--list->size];
+
+	DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
+
+	return network;
 }
 
 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
-- 
2.20.1

From 33a051e0435f6e78cc936f26f3b9ee16b7851025 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 20:00:09 +0000
Subject: [PATCH 044/111] network: Add new subnet function

The old one is too difficult to use in terms of order
of input parameters and return value.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym    |  1 +
 src/loc/network.h |  1 +
 src/network.c     | 15 +++++++++++++++
 3 files changed, 17 insertions(+)

diff --git a/src/libloc.sym b/src/libloc.sym
index c0b6b1f..5392437 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -87,6 +87,7 @@ global:
 	loc_network_get_asn;
 	loc_network_get_country_code;
 	loc_network_has_flag;
+	loc_network_is_subnet;
 	loc_network_is_subnet_of;
 	loc_network_match_asn;
 	loc_network_match_country_code;
diff --git a/src/loc/network.h b/src/loc/network.h
index e30d91c..2154cdc 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -56,6 +56,7 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
 
 int loc_network_eq(struct loc_network* self, struct loc_network* other);
 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
+int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
 struct loc_network_list* loc_network_subnets(struct loc_network* network);
 struct loc_network_list* loc_network_exclude(
diff --git a/src/network.c b/src/network.c
index d41e873..5719111 100644
--- a/src/network.c
+++ b/src/network.c
@@ -479,6 +479,21 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
 	return 0;
 }
 
+LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
+	// If the start address of the other network is smaller than this network,
+	// it cannot be a subnet.
+	if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
+		return 0;
+
+	// If the end address of the other network is greater than this network,
+	// it cannot be a subnet.
+	if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
+		return 0;
+
+	return 1;
+}
+
+// XXX DEPRECATED - I find this too difficult to use
 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
 	// If the start address of the other network is smaller than this network,
 	// it cannot be a subnet.
-- 
2.20.1

From add5bb652ba1dad1127f79cb6a0db2d363a6d5e5 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 20:01:17 +0000
Subject: [PATCH 045/111] network: Add function to exclude multiple networks at
 once

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym    |  1 +
 src/loc/network.h |  2 ++
 src/network.c     | 85 +++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 88 insertions(+)

diff --git a/src/libloc.sym b/src/libloc.sym
index 5392437..bcd11be 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -82,6 +82,7 @@ global:
 	loc_network_address_family;
 	loc_network_eq;
 	loc_network_exclude;
+	loc_network_exclude_list;
 	loc_network_format_first_address;
 	loc_network_format_last_address;
 	loc_network_get_asn;
diff --git a/src/loc/network.h b/src/loc/network.h
index 2154cdc..40712b9 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -61,6 +61,8 @@ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other
 struct loc_network_list* loc_network_subnets(struct loc_network* network);
 struct loc_network_list* loc_network_exclude(
 		struct loc_network* self, struct loc_network* other);
+struct loc_network_list* loc_network_exclude_list(
+		struct loc_network* network, struct loc_network_list* list);
 
 // List
 struct loc_network_list;
diff --git a/src/network.c b/src/network.c
index 5719111..751e8e5 100644
--- a/src/network.c
+++ b/src/network.c
@@ -677,6 +677,91 @@ ERROR:
 	return NULL;
 }
 
+LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
+		struct loc_network* network, struct loc_network_list* list) {
+	struct loc_network_list* to_check;
+
+	// Create a new list with all networks to look at
+	int r = loc_network_list_new(network->ctx, &to_check);
+	if (r)
+		return NULL;
+
+	struct loc_network* subnet = NULL;
+	struct loc_network_list* subnets = NULL;
+
+	for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
+		subnet = loc_network_list_get(list, i);
+
+		// Find all excluded networks
+		struct loc_network_list* excluded = loc_network_exclude(network, subnet);
+		if (excluded) {
+			// Add them all to the "to check" list
+			loc_network_list_merge(to_check, excluded);
+			loc_network_list_unref(excluded);
+		}
+
+		// Cleanup
+		loc_network_unref(subnet);
+	}
+
+	r = loc_network_list_new(network->ctx, &subnets);
+	if (r) {
+		loc_network_list_unref(to_check);
+		return NULL;
+	}
+
+	while (!loc_network_list_empty(to_check)) {
+		struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
+
+		// Marks whether this subnet passed all checks
+		int passed = 1;
+
+		for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
+			subnet = loc_network_list_get(list, i);
+
+			// Drop this subnet if is is already in list
+			if (loc_network_eq(subnet_to_check, subnet)) {
+				passed = 0;
+				loc_network_unref(subnet);
+				break;
+			}
+
+			// Drop this subnet if is a subnet of another subnet
+			if (loc_network_is_subnet_of(subnet, subnet_to_check)) {
+				passed = 0;
+				loc_network_unref(subnet);
+				break;
+			}
+
+			// Break it down if it overlaps
+			if (loc_network_overlaps(subnet_to_check, subnet)) {
+				passed = 0;
+
+				struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet);
+				if (excluded) {
+					loc_network_list_merge(to_check, excluded);
+					loc_network_list_unref(excluded);
+				}
+
+				loc_network_unref(subnet);
+				break;
+			}
+
+			loc_network_unref(subnet);
+		}
+
+		if (passed) {
+			r = loc_network_list_push(subnets, subnet_to_check);
+		}
+
+		loc_network_unref(subnet_to_check);
+	}
+
+	loc_network_list_unref(to_check);
+
+	return subnets;
+}
+
 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
 	// Add country code
 	loc_country_code_copy(dbobj->country_code, network->country_code);
-- 
2.20.1

From d87fd7a3d277b4b03222c7d1680e51b3e45e525b Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 20:02:03 +0000
Subject: [PATCH 046/111] database: Add option to return networks flattened

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 174 ++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 143 insertions(+), 31 deletions(-)

diff --git a/src/database.c b/src/database.c
index 9baab33..7a3d1a7 100644
--- a/src/database.c
+++ b/src/database.c
@@ -118,6 +118,9 @@ struct loc_database_enumerator {
 	struct loc_node_stack network_stack[MAX_STACK_DEPTH];
 	int network_stack_depth;
 	unsigned int* networks_visited;
+
+	// For subnet search
+	struct loc_network_list* stack;
 };
 
 static int loc_database_read_magic(struct loc_database* db) {
@@ -935,6 +938,26 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
 
 // Enumerator
 
+static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
+	DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
+
+	// Release all references
+	loc_database_unref(enumerator->db);
+	loc_unref(enumerator->ctx);
+
+	if (enumerator->string)
+		free(enumerator->string);
+
+	// Free network search
+	free(enumerator->networks_visited);
+
+	// Free subnet stack
+	if (enumerator->stack)
+		loc_network_list_unref(enumerator->stack);
+
+	free(enumerator);
+}
+
 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
 		struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
 	struct loc_database_enumerator* e = calloc(1, sizeof(*e));
@@ -951,10 +974,16 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
 	e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
 
 	// Initialise graph search
-	//e->network_stack[++e->network_stack_depth] = 0;
 	e->network_stack_depth = 1;
 	e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
 
+	// Allocate stack
+	int r = loc_network_list_new(e->ctx, &e->stack);
+	if (r) {
+		loc_database_enumerator_free(e);
+		return r;
+	}
+
 	DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
 
 	*enumerator = e;
@@ -967,22 +996,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
 	return enumerator;
 }
 
-static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
-	DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
-
-	// Release all references
-	loc_database_unref(enumerator->db);
-	loc_unref(enumerator->ctx);
-
-	if (enumerator->string)
-		free(enumerator->string);
-
-	// Free network search
-	free(enumerator->networks_visited);
-
-	free(enumerator);
-}
-
 LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
 	if (!enumerator)
 		return NULL;
@@ -1116,17 +1129,13 @@ static int loc_database_enumerator_stack_push_node(
 	return 0;
 }
 
-LOC_EXPORT int loc_database_enumerator_next_network(
-		struct loc_database_enumerator* enumerator, struct loc_network** network) {
-	// Reset network
-	*network = NULL;
-
-	// Do not do anything if not in network mode
-	if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
+static int __loc_database_enumerator_next_network(
+		struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
+	// Return top element from the stack
+	*network = loc_network_list_pop(enumerator->stack);
+	if (*network)
 		return 0;
 
-	int r;
-
 	DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
 		enumerator->network_stack_depth);
 
@@ -1155,7 +1164,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
 			enumerator->db->network_nodes_v1 + node->offset;
 
 		// Add edges to stack
-		r = loc_database_enumerator_stack_push_node(enumerator,
+		int r = loc_database_enumerator_stack_push_node(enumerator,
 			be32toh(n->one), 1, node->depth + 1);
 
 		if (r)
@@ -1181,6 +1190,10 @@ LOC_EXPORT int loc_database_enumerator_next_network(
 			if (r)
 				return r;
 
+			// Return all networks when the filter is disabled
+			if (!filter)
+				return 0;
+
 			// Check if we are interested in this network
 
 			// Skip if the family does not match
@@ -1223,12 +1236,111 @@ LOC_EXPORT int loc_database_enumerator_next_network(
 	}
 
 	// Reached the end of the search
+	return 0;
+}
 
-	// Mark all nodes as non-visited
-	for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
-		enumerator->networks_visited[i] = 0;
+static int __loc_database_enumerator_next_network_flattened(
+		struct loc_database_enumerator* enumerator, struct loc_network** network) {
+	// Fetch the next network
+	int r = __loc_database_enumerator_next_network(enumerator, network, 1);
+	if (r)
+		return r;
 
-	return 0;
+	// End if we could not read another network
+	if (!*network)
+		return 0;
+
+	struct loc_network* subnet = NULL;
+	struct loc_network_list* subnets;
+
+	// Create a list with all subnets
+	r = loc_network_list_new(enumerator->ctx, &subnets);
+	if (r)
+		return r;
+
+	// Search all subnets from the database
+	while (1) {
+		// Fetch the next network in line
+		r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
+		if (r)
+			goto END;
+
+		// End if we did not receive another subnet
+		if (!subnet)
+			break;
+
+		// Collect all subnets in a list
+		if (loc_network_is_subnet(*network, subnet)) {
+			r = loc_network_list_push(subnets, subnet);
+			if (r)
+				goto END;
+
+			loc_network_unref(subnet);
+			continue;
+		}
+
+		// If this is not a subnet, we push it back onto the stack and break
+		r = loc_network_list_push(enumerator->stack, subnet);
+		if (r)
+			goto END;
+
+		loc_network_unref(subnet);
+		break;
+	}
+
+	DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
+
+	// We can abort here if the network has no subnets
+	if (loc_network_list_empty(subnets)) {
+		loc_network_list_unref(subnets);
+
+		return 0;
+	}
+
+	// If the network has any subnets, we will break it into smaller parts
+	// without the subnets.
+	struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
+	if (!excluded || loc_network_list_empty(excluded)) {
+		r = 1;
+		goto END;
+	}
+
+	// Sort the result
+	loc_network_list_sort(excluded);
+
+	// Reverse the list
+	loc_network_list_reverse(excluded);
+
+	// Replace network with the first one
+	loc_network_unref(*network);
+
+	*network = loc_network_list_pop(excluded);
+
+	// Push the rest onto the stack
+	loc_network_list_merge(enumerator->stack, excluded);
+
+	loc_network_list_unref(excluded);
+
+END:
+	if (subnet)
+		loc_network_unref(subnet);
+
+	loc_network_list_unref(subnets);
+
+	return r;
+}
+
+LOC_EXPORT int loc_database_enumerator_next_network(
+		struct loc_database_enumerator* enumerator, struct loc_network** network) {
+	// Do not do anything if not in network mode
+	if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
+		return 0;
+
+	// Flatten output?
+	if (enumerator->flatten)
+		return __loc_database_enumerator_next_network_flattened(enumerator, network);
+
+	return __loc_database_enumerator_next_network(enumerator, network, 1);
 }
 
 LOC_EXPORT int loc_database_enumerator_next_country(
-- 
2.20.1

From d3ae93c27dcd7f6984fdc29cc141621e277f2e2a Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 20:09:20 +0000
Subject: [PATCH 047/111] test: Update API

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/test-as.c       | 2 +-
 src/test-database.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test-as.c b/src/test-as.c
index 839a04c..2d61675 100644
--- a/src/test-as.c
+++ b/src/test-as.c
@@ -95,7 +95,7 @@ int main(int argc, char** argv) {
 	// Enumerator
 
 	struct loc_database_enumerator* enumerator;
-	err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
+	err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
 	if (err) {
 		fprintf(stderr, "Could not create a database enumerator\n");
 		exit(EXIT_FAILURE);
diff --git a/src/test-database.c b/src/test-database.c
index 4aef94e..da4b11c 100644
--- a/src/test-database.c
+++ b/src/test-database.c
@@ -198,7 +198,7 @@ int main(int argc, char** argv) {
 
 	// Enumerator
 	struct loc_database_enumerator* enumerator;
-	err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
+	err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
 	if (err) {
 		fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
 		exit(EXIT_FAILURE);
-- 
2.20.1

From 594ca328c6e124d0f1eb543e9c8d9bbfe8a7b628 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 12 Nov 2020 20:09:37 +0000
Subject: [PATCH 048/111] networks: Copy all attributes when splitting networks

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/network.c b/src/network.c
index 751e8e5..d67f116 100644
--- a/src/network.c
+++ b/src/network.c
@@ -550,6 +550,20 @@ LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* netw
 	if (r)
 		goto ERROR;
 
+	// Copy country code
+	const char* country_code = loc_network_get_country_code(network);
+	if (country_code) {
+		loc_network_set_country_code(subnet1, country_code);
+		loc_network_set_country_code(subnet2, country_code);
+	}
+
+	// Copy ASN
+	uint32_t asn = loc_network_get_asn(network);
+	if (asn) {
+		loc_network_set_asn(subnet1, asn);
+		loc_network_set_asn(subnet2, asn);
+	}
+
 	loc_network_unref(subnet1);
 	loc_network_unref(subnet2);
 
-- 
2.20.1

From 69248038292e9ea1a4ee8912cdfc8700456753ad Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 13 Nov 2020 11:23:33 +0000
Subject: [PATCH 049/111] database: Move network filtering into a separate
 function

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 56 +++++++++++++++++++++++---------------------------
 1 file changed, 26 insertions(+), 30 deletions(-)

diff --git a/src/database.c b/src/database.c
index 7a3d1a7..72bc8eb 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1129,6 +1129,31 @@ static int loc_database_enumerator_stack_push_node(
 	return 0;
 }
 
+static int loc_database_enumerator_filter_network(
+		struct loc_database_enumerator* enumerator, struct loc_network* network) {
+	// Skip if the family does not match
+	if (enumerator->family && loc_network_address_family(network) != enumerator->family)
+		return 1;
+
+	// Skip if the country code does not match
+	if (*enumerator->country_code &&
+			!loc_network_match_country_code(network, enumerator->country_code))
+		return 1;
+
+	// Skip if the ASN does not match
+	if (enumerator->asn &&
+			!loc_network_match_asn(network, enumerator->asn))
+		return 1;
+
+	// Skip if flags do not match
+	if (enumerator->flags &&
+			!loc_network_match_flag(network, enumerator->flags))
+		return 1;
+
+	// Do not filter
+	return 0;
+}
+
 static int __loc_database_enumerator_next_network(
 		struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
 	// Return top element from the stack
@@ -1195,36 +1220,7 @@ static int __loc_database_enumerator_next_network(
 				return 0;
 
 			// Check if we are interested in this network
-
-			// Skip if the family does not match
-			if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
-				loc_network_unref(*network);
-				*network = NULL;
-
-				continue;
-			}
-
-			// Skip if the country code does not match
-			if (*enumerator->country_code &&
-					!loc_network_match_country_code(*network, enumerator->country_code)) {
-				loc_network_unref(*network);
-				*network = NULL;
-
-				continue;
-			}
-
-			// Skip if the ASN does not match
-			if (enumerator->asn &&
-					!loc_network_match_asn(*network, enumerator->asn)) {
-				loc_network_unref(*network);
-				*network = NULL;
-
-				continue;
-			}
-
-			// Skip if flags do not match
-			if (enumerator->flags &&
-					!loc_network_match_flag(*network, enumerator->flags)) {
+			if (loc_database_enumerator_filter_network(enumerator, *network)) {
 				loc_network_unref(*network);
 				*network = NULL;
 
-- 
2.20.1

From 2113e71bf7b997c82670c5c22cf91aa6442fe6f3 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 13 Nov 2020 11:29:02 +0000
Subject: [PATCH 050/111] database: Filter results coming from stack

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/database.c b/src/database.c
index 72bc8eb..0f3cdc2 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1157,9 +1157,23 @@ static int loc_database_enumerator_filter_network(
 static int __loc_database_enumerator_next_network(
 		struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
 	// Return top element from the stack
-	*network = loc_network_list_pop(enumerator->stack);
-	if (*network)
+	while (1) {
+		*network = loc_network_list_pop(enumerator->stack);
+
+		// Stack is empty
+		if (!*network)
+			break;
+
+		// Throw away any networks by filter
+		if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
+			loc_network_unref(*network);
+			*network = NULL;
+			continue;
+		}
+
+		// Return result
 		return 0;
+	}
 
 	DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
 		enumerator->network_stack_depth);
-- 
2.20.1

From d33753688138c9938743dafbbdddf220dd2afd14 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 13 Nov 2020 11:29:15 +0000
Subject: [PATCH 051/111] network: Sort result of excluded lists

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 3 ---
 src/network.c  | 3 +++
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/database.c b/src/database.c
index 0f3cdc2..6849d97 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1315,9 +1315,6 @@ static int __loc_database_enumerator_next_network_flattened(
 		goto END;
 	}
 
-	// Sort the result
-	loc_network_list_sort(excluded);
-
 	// Reverse the list
 	loc_network_list_reverse(excluded);
 
diff --git a/src/network.c b/src/network.c
index d67f116..9d02bf8 100644
--- a/src/network.c
+++ b/src/network.c
@@ -773,6 +773,9 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
 
 	loc_network_list_unref(to_check);
 
+	// Sort the result
+	loc_network_list_sort(subnets);
+
 	return subnets;
 }
 
-- 
2.20.1

From 8d777f12f7ffa4df1b28d197563888296803b727 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 13 Nov 2020 11:38:15 +0000
Subject: [PATCH 052/111] network: Add function to pop first element from stack

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c    |  6 ++----
 src/libloc.sym    |  1 +
 src/loc/network.h |  1 +
 src/network.c     | 19 +++++++++++++++++++
 4 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/src/database.c b/src/database.c
index 6849d97..b9d870f 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1315,15 +1315,13 @@ static int __loc_database_enumerator_next_network_flattened(
 		goto END;
 	}
 
-	// Reverse the list
-	loc_network_list_reverse(excluded);
-
 	// Replace network with the first one
 	loc_network_unref(*network);
 
-	*network = loc_network_list_pop(excluded);
+	*network = loc_network_list_pop_first(excluded);
 
 	// Push the rest onto the stack
+	loc_network_list_reverse(excluded);
 	loc_network_list_merge(enumerator->stack, excluded);
 
 	loc_network_list_unref(excluded);
diff --git a/src/libloc.sym b/src/libloc.sym
index bcd11be..6139db6 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -113,6 +113,7 @@ global:
 	loc_network_list_merge;
 	loc_network_list_new;
 	loc_network_list_pop;
+	loc_network_list_pop_first;
 	loc_network_list_push;
 	loc_network_list_ref;
 	loc_network_list_reverse;
diff --git a/src/loc/network.h b/src/loc/network.h
index 40712b9..203e61c 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -76,6 +76,7 @@ void loc_network_list_dump(struct loc_network_list* list);
 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
 int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
 void loc_network_list_sort(struct loc_network_list* list);
 void loc_network_list_reverse(struct loc_network_list* list);
diff --git a/src/network.c b/src/network.c
index 9d02bf8..e7dc97e 100644
--- a/src/network.c
+++ b/src/network.c
@@ -1231,6 +1231,25 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
 	return network;
 }
 
+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
+	// Return nothing when empty
+	if (loc_network_list_empty(list)) {
+		DEBUG(list->ctx, "%p: Popped empty stack\n", list);
+		return NULL;
+	}
+
+	struct loc_network* network = list->list[0];
+
+	// Move all elements to the top of the stack
+	for (unsigned int i = 0; i < --list->size; i++) {
+		list->list[i] = list->list[i+1];
+	}
+
+	DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
+
+	return network;
+}
+
 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
 	for (unsigned int i = 0; i < list->size; i++) {
 		if (loc_network_eq(list->list[i], network))
-- 
2.20.1

From 7933f5bfb4dd7603cb646e192840762bf6394292 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 13 Nov 2020 11:43:53 +0000
Subject: [PATCH 053/111] network: Unexport all tree functions

These should not be exported

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c      | 28 ++++++++++++++--------------
 src/test-network.c |  9 +++++++++
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/src/network.c b/src/network.c
index e7dc97e..d015579 100644
--- a/src/network.c
+++ b/src/network.c
@@ -847,7 +847,7 @@ struct loc_network_tree_node {
 	struct loc_network* network;
 };
 
-LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
+int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
 	struct loc_network_tree* t = calloc(1, sizeof(*t));
 	if (!t)
 		return -ENOMEM;
@@ -867,7 +867,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
 	return 0;
 }
 
-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
+struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
 	return loc_network_tree_node_ref(tree->root);
 }
 
@@ -939,7 +939,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
 	return 0;
 }
 
-LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
+int loc_network_tree_walk(struct loc_network_tree* tree,
 		int(*filter_callback)(struct loc_network* network, void* data),
 		int(*callback)(struct loc_network* network, void* data), void* data) {
 	return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
@@ -954,7 +954,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
 	free(tree);
 }
 
-LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
+struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
 	if (--tree->refcount > 0)
 		return tree;
 
@@ -975,13 +975,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
 	return 0;
 }
 
-LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
+int loc_network_tree_dump(struct loc_network_tree* tree) {
 	DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
 
 	return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
 }
 
-LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
+int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
 	DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
 
 	struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
@@ -1012,7 +1012,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
 	return 0;
 }
 
-LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
+size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
 	size_t counter = 0;
 
 	int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
@@ -1034,11 +1034,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
 	return counter;
 }
 
-LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
+size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
 	return __loc_network_tree_count_nodes(tree->root);
 }
 
-LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
+int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
 	struct loc_network_tree_node* n = calloc(1, sizeof(*n));
 	if (!n)
 		return -ENOMEM;
@@ -1053,7 +1053,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
 	return 0;
 }
 
-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
+struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
 	if (node)
 		node->refcount++;
 
@@ -1076,7 +1076,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
 	free(node);
 }
 
-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
+struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
 	if (!node)
 		return NULL;
 
@@ -1087,7 +1087,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
 	return NULL;
 }
 
-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
+struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
 	if (index == 0)
 		node = node->zero;
 	else
@@ -1099,11 +1099,11 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
 	return loc_network_tree_node_ref(node);
 }
 
-LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
+int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
 	return (!!node->network);
 }
 
-LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
+struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
 	return loc_network_ref(node->network);
 }
 
diff --git a/src/test-network.c b/src/test-network.c
index af1b2e6..7c90224 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -37,12 +37,14 @@ int main(int argc, char** argv) {
 	// Enable debug logging
 	loc_set_log_priority(ctx, LOG_DEBUG);
 
+#if 0
 	struct loc_network_tree* tree;
 	err = loc_network_tree_new(ctx, &tree);
 	if (err) {
 		fprintf(stderr, "Could not create the network tree\n");
 		exit(EXIT_FAILURE);
 	}
+#endif
 
 	// Create a network
 	struct loc_network* network1;
@@ -58,12 +60,14 @@ int main(int argc, char** argv) {
 		exit(EXIT_FAILURE);
 	}
 
+#if 0
 	// Adding network to the tree
 	err = loc_network_tree_add_network(tree, network1);
 	if (err) {
 		fprintf(stderr, "Could not add network to the tree\n");
 		exit(EXIT_FAILURE);
 	}
+#endif
 
 	// Check if the first and last addresses are correct
 	char* string = loc_network_format_first_address(network1);
@@ -101,6 +105,7 @@ int main(int argc, char** argv) {
 		exit(EXIT_FAILURE);
 	}
 
+#if 0
 	// Adding network to the tree
 	err = loc_network_tree_add_network(tree, network2);
 	if (err) {
@@ -117,6 +122,7 @@ int main(int argc, char** argv) {
 
 	size_t nodes = loc_network_tree_count_nodes(tree);
 	printf("The tree has %zu nodes\n", nodes);
+#endif
 
 	// Check equals function
 	err = loc_network_eq(network1, network1);
@@ -260,7 +266,10 @@ int main(int argc, char** argv) {
 	loc_network_unref(network2);
 	loc_network_unref(network3);
 	loc_network_unref(network4);
+
+#if 0
 	loc_network_tree_unref(tree);
+#endif
 
 	// And open it again from disk
 	struct loc_database* db;
-- 
2.20.1

From c242f7325bd6fc4ba26047ac24196d1c161c6e01 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 13 Nov 2020 12:09:03 +0000
Subject: [PATCH 054/111] python: Move tree flattening into C

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/database.c |   8 ++-
 src/python/export.py  | 138 +++++-------------------------------------
 2 files changed, 21 insertions(+), 125 deletions(-)

diff --git a/src/python/database.c b/src/python/database.c
index 7f8c2c2..d169547 100644
--- a/src/python/database.c
+++ b/src/python/database.c
@@ -258,17 +258,19 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
 }
 
 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
-	char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
+	char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
 	const char* country_code = NULL;
 	unsigned int asn = 0;
 	int flags = 0;
 	int family = 0;
+	int flatten = 0;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
 		return NULL;
 
 	struct loc_database_enumerator* enumerator;
-	int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
+	int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
+		(flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
 	if (r) {
 		PyErr_SetFromErrno(PyExc_SystemError);
 		return NULL;
diff --git a/src/python/export.py b/src/python/export.py
index dd44332..be4a68e 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -87,58 +87,12 @@ class OutputWriter(object):
 	def _write_network(self, network):
 		self.f.write("%s\n" % network)
 
-	def write(self, network, subnets):
+	def write(self, network):
 		if self.flatten and self._flatten(network):
 			log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
 			return
 
-		# Convert network into a Python object
-		_network = ipaddress.ip_network("%s" % network)
-
-		# Write the network when it has no subnets
-		if not subnets:
-			log.debug("Writing %s to %s" % (_network, self.f))
-			return self._write_network(_network)
-
-		# Convert subnets into Python objects
-		_subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
-
-		# Split the network into smaller bits so that
-		# we can accomodate for any gaps in it later
-		to_check = set()
-		for _subnet in _subnets:
-			to_check.update(
-				_network.address_exclude(_subnet)
-			)
-
-		# Clear the list of all subnets
-		subnets = []
-
-		# Check if all subnets to not overlap with anything else
-		while to_check:
-			subnet_to_check = to_check.pop()
-
-			for _subnet in _subnets:
-				# Drop this subnet if it equals one of the subnets
-				# or if it is subnet of one of them
-				if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
-					break
-
-				# Break it down if it overlaps
-				if subnet_to_check.overlaps(_subnet):
-					to_check.update(
-						subnet_to_check.address_exclude(_subnet)
-					)
-					break
-
-			# Add the subnet again as it passed the check
-			else:
-				subnets.append(subnet_to_check)
-
-		# Write all networks as compact as possible
-		for network in ipaddress.collapse_addresses(subnets):
-			log.debug("Writing %s to %s" % (network, self.f))
-			self._write_network(network)
+		return self._write_network(network)
 
 	def finish(self):
 		"""
@@ -188,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter):
 	mode = "wb"
 
 	def _write_network(self, network):
-		for address in (network.network_address, network.broadcast_address):
+		for address in (network.first_address, network.last_address):
 			# Convert this into a string of bits
 			bytes = socket.inet_pton(
 				socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
@@ -231,42 +185,21 @@ class Exporter(object):
 				writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
 
 			# Get all networks that match the family
-			networks = self.db.search_networks(family=family)
-
-			# Create a stack with all networks in order where we can put items back
-			# again and retrieve them in the next iteration.
-			networks = BufferedStack(networks)
+			networks = self.db.search_networks(family=family, flatten=True)
 
 			# Walk through all networks
 			for network in networks:
-				# Collect all networks which are a subnet of network
-				subnets = []
-				for subnet in networks:
-					# If the next subnet was not a subnet, we have to push
-					# it back on the stack and break this loop
-					if not subnet.is_subnet_of(network):
-						networks.push(subnet)
-						break
-
-					subnets.append(subnet)
-
 				# Write matching countries
-				if network.country_code and network.country_code in writers:
-					# Mismatching subnets
-					gaps = [
-						subnet for subnet in subnets if not network.country_code == subnet.country_code
-					]
-
-					writers[network.country_code].write(network, gaps)
+				try:
+					writers[network.country_code].write(network)
+				except KeyError:
+					pass
 
 				# Write matching ASNs
-				if network.asn and network.asn in writers:
-					# Mismatching subnets
-					gaps = [
-						subnet for subnet in subnets if not network.asn == subnet.asn
-					]
-
-					writers[network.asn].write(network, gaps)
+				try:
+					writers[network.asn].write(network)
+				except KeyError:
+					pass
 
 				# Handle flags
 				for flag in flags:
@@ -274,19 +207,10 @@ class Exporter(object):
 						# Fetch the "fake" country code
 						country = flags[flag]
 
-						if not country in writers:
-							continue
-
-						gaps = [
-							subnet for subnet in subnets
-								if not subnet.has_flag(flag)
-						]
-
-						writers[country].write(network, gaps)
-
-				# Push all subnets back onto the stack
-				for subnet in reversed(subnets):
-					networks.push(subnet)
+						try:
+							writers[country].write(network)
+						except KeyError:
+							pass
 
 			# Write everything to the filesystem
 			for writer in writers.values():
@@ -298,33 +222,3 @@ class Exporter(object):
 		)
 
 		return os.path.join(directory, filename)
-
-
-class BufferedStack(object):
-	"""
-		This class takes an iterator and when being iterated
-		over it returns objects from that iterator for as long
-		as there are any.
-
-		It additionally has a function to put an item back on
-		the back so that it will be returned again at the next
-		iteration.
-	"""
-	def __init__(self, iterator):
-		self.iterator = iterator
-		self.stack = []
-
-	def __iter__(self):
-		return self
-
-	def __next__(self):
-		if self.stack:
-			return self.stack.pop(0)
-
-		return next(self.iterator)
-
-	def push(self, elem):
-		"""
-			Takes an element and puts it on the stack
-		"""
-		self.stack.insert(0, elem)
-- 
2.20.1

From e0b9ff5f38beb0d560b16db881647e5a75127df1 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Sun, 15 Nov 2020 15:02:28 +0000
Subject: [PATCH 055/111] Move network lists into an own file

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 Makefile.am            |   2 +
 src/libloc.sym         |   1 +
 src/loc/network-list.h |  37 +++++++
 src/loc/network.h      |  20 +---
 src/network-list.c     | 224 +++++++++++++++++++++++++++++++++++++++++
 src/network.c          | 207 +------------------------------------
 src/python/network.c   |   1 +
 7 files changed, 269 insertions(+), 223 deletions(-)
 create mode 100644 src/loc/network-list.h
 create mode 100644 src/network-list.c

diff --git a/Makefile.am b/Makefile.am
index a0431a6..f0d8c4c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -96,6 +96,7 @@ pkginclude_HEADERS = \
 	src/loc/database.h \
 	src/loc/format.h \
 	src/loc/network.h \
+	src/loc/network-list.h \
 	src/loc/private.h \
 	src/loc/stringpool.h \
 	src/loc/resolv.h \
@@ -110,6 +111,7 @@ src_libloc_la_SOURCES = \
 	src/country.c \
 	src/database.c \
 	src/network.c \
+	src/network-list.c \
 	src/resolv.c \
 	src/stringpool.c \
 	src/writer.c
diff --git a/src/libloc.sym b/src/libloc.sym
index 6139db6..453a1be 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -87,6 +87,7 @@ global:
 	loc_network_format_last_address;
 	loc_network_get_asn;
 	loc_network_get_country_code;
+	loc_network_gt;
 	loc_network_has_flag;
 	loc_network_is_subnet;
 	loc_network_is_subnet_of;
diff --git a/src/loc/network-list.h b/src/loc/network-list.h
new file mode 100644
index 0000000..af3b28d
--- /dev/null
+++ b/src/loc/network-list.h
@@ -0,0 +1,37 @@
+/*
+	libloc - A library to determine the location of someone on the Internet
+
+	Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
+
+	This library is free software; you can redistribute it and/or
+	modify it under the terms of the GNU Lesser General Public
+	License as published by the Free Software Foundation; either
+	version 2.1 of the License, or (at your option) any later version.
+
+	This library is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+	Lesser General Public License for more details.
+*/
+
+#ifndef LIBLOC_NETWORK_LIST_H
+#define LIBLOC_NETWORK_LIST_H
+
+struct loc_network_list;
+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
+size_t loc_network_list_size(struct loc_network_list* list);
+int loc_network_list_empty(struct loc_network_list* list);
+void loc_network_list_clear(struct loc_network_list* list);
+void loc_network_list_dump(struct loc_network_list* list);
+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
+struct loc_network* loc_network_list_pop(struct loc_network_list* list);
+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
+void loc_network_list_sort(struct loc_network_list* list);
+void loc_network_list_reverse(struct loc_network_list* list);
+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
+
+#endif
diff --git a/src/loc/network.h b/src/loc/network.h
index 203e61c..d86b685 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -21,6 +21,7 @@
 
 #include <loc/libloc.h>
 #include <loc/format.h>
+#include <loc/network-list.h>
 
 enum loc_network_flags {
 	LOC_NETWORK_FLAG_ANONYMOUS_PROXY    = (1 << 0), // A1
@@ -55,6 +56,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
 
 int loc_network_eq(struct loc_network* self, struct loc_network* other);
+int loc_network_gt(struct loc_network* self, struct loc_network* other);
 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
@@ -64,24 +66,6 @@ struct loc_network_list* loc_network_exclude(
 struct loc_network_list* loc_network_exclude_list(
 		struct loc_network* network, struct loc_network_list* list);
 
-// List
-struct loc_network_list;
-int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
-struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
-struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
-size_t loc_network_list_size(struct loc_network_list* list);
-int loc_network_list_empty(struct loc_network_list* list);
-void loc_network_list_clear(struct loc_network_list* list);
-void loc_network_list_dump(struct loc_network_list* list);
-struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
-int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
-struct loc_network* loc_network_list_pop(struct loc_network_list* list);
-struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
-int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
-void loc_network_list_sort(struct loc_network_list* list);
-void loc_network_list_reverse(struct loc_network_list* list);
-int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
-
 #ifdef LIBLOC_PRIVATE
 
 int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
diff --git a/src/network-list.c b/src/network-list.c
new file mode 100644
index 0000000..1f6e80e
--- /dev/null
+++ b/src/network-list.c
@@ -0,0 +1,224 @@
+/*
+	libloc - A library to determine the location of someone on the Internet
+
+	Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
+
+	This library is free software; you can redistribute it and/or
+	modify it under the terms of the GNU Lesser General Public
+	License as published by the Free Software Foundation; either
+	version 2.1 of the License, or (at your option) any later version.
+
+	This library is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+	Lesser General Public License for more details.
+*/
+
+#include <errno.h>
+#include <stdlib.h>
+
+#include <loc/libloc.h>
+#include <loc/network.h>
+#include <loc/private.h>
+
+struct loc_network_list {
+	struct loc_ctx* ctx;
+	int refcount;
+
+	struct loc_network* list[1024];
+	size_t size;
+	size_t max_size;
+};
+
+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
+		struct loc_network_list** list) {
+	struct loc_network_list* l = calloc(1, sizeof(*l));
+	if (!l)
+		return -ENOMEM;
+
+	l->ctx = loc_ref(ctx);
+	l->refcount = 1;
+
+	// Do not allow this list to grow larger than this
+	l->max_size = 1024;
+
+	DEBUG(l->ctx, "Network list allocated at %p\n", l);
+	*list = l;
+	return 0;
+}
+
+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
+	list->refcount++;
+
+	return list;
+}
+
+static void loc_network_list_free(struct loc_network_list* list) {
+	DEBUG(list->ctx, "Releasing network list at %p\n", list);
+
+	for (unsigned int i = 0; i < list->size; i++)
+		loc_network_unref(list->list[i]);
+
+	loc_unref(list->ctx);
+	free(list);
+}
+
+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
+	if (!list)
+		return NULL;
+
+	if (--list->refcount > 0)
+		return list;
+
+	loc_network_list_free(list);
+	return NULL;
+}
+
+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
+	return list->size;
+}
+
+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
+	return list->size == 0;
+}
+
+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
+	for (unsigned int i = 0; i < list->size; i++)
+		loc_network_unref(list->list[i]);
+
+	list->size = 0;
+}
+
+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
+	struct loc_network* network;
+	char* s;
+
+	for (unsigned int i = 0; i < list->size; i++) {
+		network = list->list[i];
+
+		s = loc_network_str(network);
+
+		INFO(list->ctx, "%s\n", s);
+		free(s);
+	}
+}
+
+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
+	// Check index
+	if (index >= list->size)
+		return NULL;
+
+	return loc_network_ref(list->list[index]);
+}
+
+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
+	// Do not add networks that are already on the list
+	if (loc_network_list_contains(list, network))
+		return 0;
+
+	// Check if we have space left
+	if (list->size == list->max_size) {
+		ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
+		return -ENOMEM;
+	}
+
+	DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
+
+	list->list[list->size++] = loc_network_ref(network);
+
+	return 0;
+}
+
+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
+	// Return nothing when empty
+	if (loc_network_list_empty(list)) {
+		DEBUG(list->ctx, "%p: Popped empty stack\n", list);
+		return NULL;
+	}
+
+	struct loc_network* network = list->list[--list->size];
+
+	DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
+
+	return network;
+}
+
+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
+	// Return nothing when empty
+	if (loc_network_list_empty(list)) {
+		DEBUG(list->ctx, "%p: Popped empty stack\n", list);
+		return NULL;
+	}
+
+	struct loc_network* network = list->list[0];
+
+	// Move all elements to the top of the stack
+	for (unsigned int i = 0; i < --list->size; i++) {
+		list->list[i] = list->list[i+1];
+	}
+
+	DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
+
+	return network;
+}
+
+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
+	for (unsigned int i = 0; i < list->size; i++) {
+		if (loc_network_eq(list->list[i], network))
+			return 1;
+	}
+
+	return 0;
+}
+
+static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
+	// Do nothing for invalid indices
+	if (i1 >= list->size || i2 >= list->size)
+		return;
+
+	struct loc_network* network1 = list->list[i1];
+	struct loc_network* network2 = list->list[i2];
+
+	list->list[i1] = network2;
+	list->list[i2] = network1;
+}
+
+LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
+	unsigned int i = 0;
+	unsigned int j = list->size - 1;
+
+	while (i < j) {
+		loc_network_list_swap(list, i++, j--);
+	}
+}
+
+LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
+	unsigned int n = list->size;
+	int swapped;
+
+	do {
+		swapped = 0;
+
+		for (unsigned int i = 1; i < n; i++) {
+			if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
+				loc_network_list_swap(list, i-1, i);
+				swapped = 1;
+			}
+		}
+
+		n--;
+	} while (swapped);
+}
+
+LOC_EXPORT int loc_network_list_merge(
+		struct loc_network_list* self, struct loc_network_list* other) {
+	int r;
+
+	for (unsigned int i = 0; i < other->size; i++) {
+		r = loc_network_list_push(self, other->list[i]);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
diff --git a/src/network.c b/src/network.c
index d015579..28ca2df 100644
--- a/src/network.c
+++ b/src/network.c
@@ -29,6 +29,7 @@
 #include <loc/compat.h>
 #include <loc/country.h>
 #include <loc/network.h>
+#include <loc/network-list.h>
 #include <loc/private.h>
 
 struct loc_network {
@@ -436,7 +437,7 @@ LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* othe
 	return 1;
 }
 
-static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
+LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
 	// Families must match
 	if (self->family != other->family)
 		return -1;
@@ -1106,207 +1107,3 @@ int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
 struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
 	return loc_network_ref(node->network);
 }
-
-// List
-
-struct loc_network_list {
-	struct loc_ctx* ctx;
-	int refcount;
-
-	struct loc_network* list[1024];
-	size_t size;
-	size_t max_size;
-};
-
-LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
-		struct loc_network_list** list) {
-	struct loc_network_list* l = calloc(1, sizeof(*l));
-	if (!l)
-		return -ENOMEM;
-
-	l->ctx = loc_ref(ctx);
-	l->refcount = 1;
-
-	// Do not allow this list to grow larger than this
-	l->max_size = 1024;
-
-	DEBUG(l->ctx, "Network list allocated at %p\n", l);
-	*list = l;
-	return 0;
-}
-
-LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
-	list->refcount++;
-
-	return list;
-}
-
-static void loc_network_list_free(struct loc_network_list* list) {
-	DEBUG(list->ctx, "Releasing network list at %p\n", list);
-
-	for (unsigned int i = 0; i < list->size; i++)
-		loc_network_unref(list->list[i]);
-
-	loc_unref(list->ctx);
-	free(list);
-}
-
-LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
-	if (!list)
-		return NULL;
-
-	if (--list->refcount > 0)
-		return list;
-
-	loc_network_list_free(list);
-	return NULL;
-}
-
-LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
-	return list->size;
-}
-
-LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
-	return list->size == 0;
-}
-
-LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
-	for (unsigned int i = 0; i < list->size; i++)
-		loc_network_unref(list->list[i]);
-
-	list->size = 0;
-}
-
-LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
-	struct loc_network* network;
-	char* s;
-
-	for (unsigned int i = 0; i < list->size; i++) {
-		network = list->list[i];
-
-		s = loc_network_str(network);
-
-		INFO(list->ctx, "%s\n", s);
-		free(s);
-	}
-}
-
-LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
-	// Check index
-	if (index >= list->size)
-		return NULL;
-
-	return loc_network_ref(list->list[index]);
-}
-
-LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
-	// Do not add networks that are already on the list
-	if (loc_network_list_contains(list, network))
-		return 0;
-
-	// Check if we have space left
-	if (list->size == list->max_size) {
-		ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
-		return -ENOMEM;
-	}
-
-	DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
-
-	list->list[list->size++] = loc_network_ref(network);
-
-	return 0;
-}
-
-LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
-	// Return nothing when empty
-	if (loc_network_list_empty(list)) {
-		DEBUG(list->ctx, "%p: Popped empty stack\n", list);
-		return NULL;
-	}
-
-	struct loc_network* network = list->list[--list->size];
-
-	DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
-
-	return network;
-}
-
-LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
-	// Return nothing when empty
-	if (loc_network_list_empty(list)) {
-		DEBUG(list->ctx, "%p: Popped empty stack\n", list);
-		return NULL;
-	}
-
-	struct loc_network* network = list->list[0];
-
-	// Move all elements to the top of the stack
-	for (unsigned int i = 0; i < --list->size; i++) {
-		list->list[i] = list->list[i+1];
-	}
-
-	DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
-
-	return network;
-}
-
-LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
-	for (unsigned int i = 0; i < list->size; i++) {
-		if (loc_network_eq(list->list[i], network))
-			return 1;
-	}
-
-	return 0;
-}
-
-static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
-	// Do nothing for invalid indices
-	if (i1 >= list->size || i2 >= list->size)
-		return;
-
-	struct loc_network* network1 = list->list[i1];
-	struct loc_network* network2 = list->list[i2];
-
-	list->list[i1] = network2;
-	list->list[i2] = network1;
-}
-
-LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
-	unsigned int i = 0;
-	unsigned int j = list->size - 1;
-
-	while (i < j) {
-		loc_network_list_swap(list, i++, j--);
-	}
-}
-
-LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
-	unsigned int n = list->size;
-	int swapped;
-
-	do {
-		swapped = 0;
-
-		for (unsigned int i = 1; i < n; i++) {
-			if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
-				loc_network_list_swap(list, i-1, i);
-				swapped = 1;
-			}
-		}
-
-		n--;
-	} while (swapped);
-}
-
-LOC_EXPORT int loc_network_list_merge(
-		struct loc_network_list* self, struct loc_network_list* other) {
-	int r;
-
-	for (unsigned int i = 0; i < other->size; i++) {
-		r = loc_network_list_push(self, other->list[i]);
-		if (r)
-			return r;
-	}
-
-	return 0;
-}
diff --git a/src/python/network.c b/src/python/network.c
index 11f672b..ed91d65 100644
--- a/src/python/network.c
+++ b/src/python/network.c
@@ -20,6 +20,7 @@
 
 #include <loc/libloc.h>
 #include <loc/network.h>
+#include <loc/network-list.h>
 
 #include "locationmodule.h"
 #include "network.h"
-- 
2.20.1

From e646a8f35ec7eff009414b3fd107c9af5cf39a86 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Mon, 16 Nov 2020 15:13:28 +0000
Subject: [PATCH 056/111] Implement filtering for multiple countries in the
 enumerator

This will allow us to speed up the export of the database
if only a few countries should be returned.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 Makefile.am            |   2 +
 src/country-list.c     | 138 +++++++++++++++++++++++++++++++++++++++++
 src/country.c          |   3 +
 src/database.c         |  47 ++++++--------
 src/libloc.sym         |  15 ++++-
 src/loc/country-list.h |  43 +++++++++++++
 src/loc/database.h     |   5 +-
 src/python/database.c  |  57 ++++++++++++++---
 8 files changed, 274 insertions(+), 36 deletions(-)
 create mode 100644 src/country-list.c
 create mode 100644 src/loc/country-list.h

diff --git a/Makefile.am b/Makefile.am
index f0d8c4c..f4ca3c8 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -93,6 +93,7 @@ pkginclude_HEADERS = \
 	src/loc/as.h \
 	src/loc/compat.h \
 	src/loc/country.h \
+	src/loc/country-list.h \
 	src/loc/database.h \
 	src/loc/format.h \
 	src/loc/network.h \
@@ -109,6 +110,7 @@ src_libloc_la_SOURCES = \
 	src/libloc.c \
 	src/as.c \
 	src/country.c \
+	src/country-list.c \
 	src/database.c \
 	src/network.c \
 	src/network-list.c \
diff --git a/src/country-list.c b/src/country-list.c
new file mode 100644
index 0000000..ae0d71a
--- /dev/null
+++ b/src/country-list.c
@@ -0,0 +1,138 @@
+/*
+	libloc - A library to determine the location of someone on the Internet
+
+	Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
+
+	This library is free software; you can redistribute it and/or
+	modify it under the terms of the GNU Lesser General Public
+	License as published by the Free Software Foundation; either
+	version 2.1 of the License, or (at your option) any later version.
+
+	This library is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+	Lesser General Public License for more details.
+*/
+
+#include <errno.h>
+#include <stdlib.h>
+
+#include <loc/country.h>
+#include <loc/country-list.h>
+#include <loc/private.h>
+
+struct loc_country_list {
+	struct loc_ctx* ctx;
+	int refcount;
+
+	struct loc_country* list[1024];
+	size_t size;
+	size_t max_size;
+};
+
+LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
+		struct loc_country_list** list) {
+	struct loc_country_list* l = calloc(1, sizeof(*l));
+	if (!l)
+		return -ENOMEM;
+
+	l->ctx = loc_ref(ctx);
+	l->refcount = 1;
+
+	// Do not allow this list to grow larger than this
+	l->max_size = 1024;
+
+	DEBUG(l->ctx, "Country list allocated at %p\n", l);
+	*list = l;
+
+	return 0;
+}
+
+LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
+	list->refcount++;
+
+	return list;
+}
+
+static void loc_country_list_free(struct loc_country_list* list) {
+	DEBUG(list->ctx, "Releasing country list at %p\n", list);
+
+	loc_country_list_clear(list);
+
+	loc_unref(list->ctx);
+	free(list);
+}
+
+LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
+	if (!list)
+		return NULL;
+
+	if (--list->refcount > 0)
+		return list;
+
+	loc_country_list_free(list);
+	return NULL;
+}
+
+LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
+	return list->size;
+}
+
+LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
+	return list->size == 0;
+}
+
+LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
+	for (unsigned int i = 0; i < list->size; i++)
+		loc_country_unref(list->list[i]);
+}
+
+LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
+	// Check index
+	if (index >= list->size)
+		return NULL;
+
+	return loc_country_ref(list->list[index]);
+}
+
+LOC_EXPORT int loc_country_list_append(
+		struct loc_country_list* list, struct loc_country* country) {
+	if (loc_country_list_contains(list, country))
+		return 0;
+
+	// Check if we have space left
+	if (list->size == list->max_size) {
+		ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
+		return -ENOMEM;
+	}
+
+	DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
+
+	list->list[list->size++] = loc_country_ref(country);
+
+	return 0;
+}
+
+LOC_EXPORT int loc_country_list_contains(
+		struct loc_country_list* list, struct loc_country* country) {
+	for (unsigned int i = 0; i < list->size; i++) {
+		if (loc_country_cmp(country, list->list[i]) == 0)
+			return 1;
+	}
+
+	return 0;
+}
+
+LOC_EXPORT int loc_country_list_contains_code(
+		struct loc_country_list* list, const char* code) {
+	struct loc_country* country;
+
+	int r = loc_country_new(list->ctx, &country, code);
+	if (r)
+		return -1;
+
+	r = loc_country_list_contains(list, country);
+	loc_country_unref(country);
+
+	return r;
+}
diff --git a/src/country.c b/src/country.c
index 2ba93e6..7aac0db 100644
--- a/src/country.c
+++ b/src/country.c
@@ -34,6 +34,9 @@ struct loc_country {
 };
 
 LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
+	if (!loc_country_code_is_valid(country_code))
+		return -EINVAL;
+
 	struct loc_country* c = calloc(1, sizeof(*c));
 	if (!c)
 		return -ENOMEM;
diff --git a/src/database.c b/src/database.c
index b9d870f..29823b2 100644
--- a/src/database.c
+++ b/src/database.c
@@ -40,6 +40,7 @@
 #include <loc/as.h>
 #include <loc/compat.h>
 #include <loc/country.h>
+#include <loc/country-list.h>
 #include <loc/database.h>
 #include <loc/format.h>
 #include <loc/network.h>
@@ -99,7 +100,7 @@ struct loc_database_enumerator {
 
 	// Search string
 	char* string;
-	char country_code[3];
+	struct loc_country_list* countries;
 	uint32_t asn;
 	enum loc_network_flags flags;
 	int family;
@@ -1017,33 +1018,20 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
 	return 0;
 }
 
-LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
-	// Set empty country code
-	if (!country_code || !*country_code) {
-		*enumerator->country_code = '\0';
-		return 0;
-	}
+LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
+		struct loc_database_enumerator* enumerator) {
+	if (!enumerator->countries)
+		return NULL;
 
-	// Treat A1, A2, A3 as special country codes,
-	// but perform search for flags instead
-	if (strcmp(country_code, "A1") == 0) {
-		return loc_database_enumerator_set_flag(enumerator,
-			LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
-	} else if (strcmp(country_code, "A2") == 0) {
-		return loc_database_enumerator_set_flag(enumerator,
-			LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
-	} else if (strcmp(country_code, "A3") == 0) {
-		return loc_database_enumerator_set_flag(enumerator,
-			LOC_NETWORK_FLAG_ANYCAST);
-	}
+	return loc_country_list_ref(enumerator->countries);
+}
 
-	// Country codes must be two characters
-	if (!loc_country_code_is_valid(country_code))
-		return -EINVAL;
+LOC_EXPORT int loc_database_enumerator_set_countries(
+		struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
+	if (enumerator->countries)
+		loc_country_list_unref(enumerator->countries);
 
-	for (unsigned int i = 0; i < 3; i++) {
-		enumerator->country_code[i] = country_code[i];
-	}
+	enumerator->countries = loc_country_list_ref(countries);
 
 	return 0;
 }
@@ -1129,6 +1117,12 @@ static int loc_database_enumerator_stack_push_node(
 	return 0;
 }
 
+static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
+	const char* country_code = loc_network_get_country_code(network);
+
+	return loc_country_list_contains_code(countries, country_code);
+}
+
 static int loc_database_enumerator_filter_network(
 		struct loc_database_enumerator* enumerator, struct loc_network* network) {
 	// Skip if the family does not match
@@ -1136,8 +1130,7 @@ static int loc_database_enumerator_filter_network(
 		return 1;
 
 	// Skip if the country code does not match
-	if (*enumerator->country_code &&
-			!loc_network_match_country_code(network, enumerator->country_code))
+	if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
 		return 1;
 
 	// Skip if the ASN does not match
diff --git a/src/libloc.sym b/src/libloc.sym
index 453a1be..40e9f88 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -49,6 +49,18 @@ global:
 	loc_country_set_name;
 	loc_country_unref;
 
+	# Country List
+	loc_country_list_append;
+	loc_country_list_clear;
+	loc_country_list_contains;
+	loc_country_list_contains_code;
+	loc_country_list_empty;
+	loc_country_list_get;
+	loc_country_list_new;
+	loc_country_list_ref;
+	loc_country_list_size;
+	loc_country_list_unref;
+
 	# Database
 	loc_database_add_as;
 	loc_database_count_as;
@@ -66,13 +78,14 @@ global:
 	loc_database_verify;
 
 	# Database Enumerator
+	loc_database_enumerator_get_countries;
 	loc_database_enumerator_new;
 	loc_database_enumerator_next_as;
 	loc_database_enumerator_next_country;
 	loc_database_enumerator_next_network;
 	loc_database_enumerator_ref;
 	loc_database_enumerator_set_asn;
-	loc_database_enumerator_set_country_code;
+	loc_database_enumerator_set_countries;
 	loc_database_enumerator_set_family;
 	loc_database_enumerator_set_flag;
 	loc_database_enumerator_set_string;
diff --git a/src/loc/country-list.h b/src/loc/country-list.h
new file mode 100644
index 0000000..a7f818a
--- /dev/null
+++ b/src/loc/country-list.h
@@ -0,0 +1,43 @@
+/*
+	libloc - A library to determine the location of someone on the Internet
+
+	Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
+
+	This library is free software; you can redistribute it and/or
+	modify it under the terms of the GNU Lesser General Public
+	License as published by the Free Software Foundation; either
+	version 2.1 of the License, or (at your option) any later version.
+
+	This library is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+	Lesser General Public License for more details.
+*/
+
+#ifndef LIBLOC_COUNTRY_LIST_H
+#define LIBLOC_COUNTRY_LIST_H
+
+#include <stdlib.h>
+
+#include <loc/libloc.h>
+#include <loc/country.h>
+
+struct loc_country_list;
+
+int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
+struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
+struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
+
+size_t loc_country_list_size(struct loc_country_list* list);
+int loc_country_list_empty(struct loc_country_list* list);
+void loc_country_list_clear(struct loc_country_list* list);
+
+struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
+int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
+
+int loc_country_list_contains(
+	struct loc_country_list* list, struct loc_country* country);
+int loc_country_list_contains_code(
+	struct loc_country_list* list, const char* code);
+
+#endif
diff --git a/src/loc/database.h b/src/loc/database.h
index 14eb5ea..246e5c5 100644
--- a/src/loc/database.h
+++ b/src/loc/database.h
@@ -25,6 +25,7 @@
 #include <loc/network.h>
 #include <loc/as.h>
 #include <loc/country.h>
+#include <loc/country-list.h>
 
 struct loc_database;
 int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
@@ -66,7 +67,9 @@ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_
 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
 
 int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
-int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
+struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
+int loc_database_enumerator_set_countries(
+	struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
 int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
diff --git a/src/python/database.c b/src/python/database.c
index d169547..e6f6f37 100644
--- a/src/python/database.c
+++ b/src/python/database.c
@@ -258,14 +258,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
 }
 
 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
-	char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
-	const char* country_code = NULL;
+	char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
+	PyObject* country_codes = NULL;
 	unsigned int asn = 0;
 	int flags = 0;
 	int family = 0;
 	int flatten = 0;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
+			&PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
 		return NULL;
 
 	struct loc_database_enumerator* enumerator;
@@ -277,13 +278,55 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
 	}
 
 	// Set country code we are searching for
-	if (country_code) {
-		r = loc_database_enumerator_set_country_code(enumerator, country_code);
-
+	if (country_codes) {
+		struct loc_country_list* countries;
+		r = loc_country_list_new(loc_ctx, &countries);
 		if (r) {
-			PyErr_SetFromErrno(PyExc_SystemError);
+			PyErr_SetString(PyExc_SystemError, "Could not create country list");
 			return NULL;
 		}
+
+		for (unsigned int i = 0; i < PyList_Size(country_codes); i++) {
+			PyObject* item = PyList_GetItem(country_codes, i);
+
+			if (!PyUnicode_Check(item)) {
+				PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
+				loc_country_list_unref(countries);
+				return NULL;
+			}
+
+			const char* country_code = PyUnicode_AsUTF8(item);
+
+			struct loc_country* country;
+			r = loc_country_new(loc_ctx, &country, country_code);
+			if (r) {
+				if (r == -EINVAL) {
+					PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
+				} else {
+					PyErr_SetString(PyExc_SystemError, "Could not create country");
+				}
+
+				loc_country_list_unref(countries);
+				return NULL;
+			}
+
+			// Append it to the list
+			r = loc_country_list_append(countries, country);
+			if (r) {
+				PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
+
+				loc_country_list_unref(countries);
+				loc_country_unref(country);
+				return NULL;
+			}
+
+			loc_country_unref(country);
+		}
+
+		loc_database_enumerator_set_countries(enumerator, countries);
+
+		Py_DECREF(country_codes);
+		loc_country_list_unref(countries);
 	}
 
 	// Set the ASN we are searching for
-- 
2.20.1

From 7af51f8a579c79714992a3e175036fb511139310 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Mon, 16 Nov 2020 15:20:50 +0000
Subject: [PATCH 057/111] python: Only return country codes we want

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/export.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/python/export.py b/src/python/export.py
index be4a68e..5e7fe53 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -184,8 +184,14 @@ class Exporter(object):
 
 				writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
 
+			# Filter countries from special country codes
+			country_codes = [
+				country_code for country_code in countries if not country_code in flags.values()
+			]
+
 			# Get all networks that match the family
-			networks = self.db.search_networks(family=family, flatten=True)
+			networks = self.db.search_networks(family=family,
+				country_codes=country_codes, flatten=True)
 
 			# Walk through all networks
 			for network in networks:
-- 
2.20.1

From bd1dc6bf6fe4ce40bf12e7426e283b31afd274e1 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Mon, 16 Nov 2020 15:25:15 +0000
Subject: [PATCH 058/111] database: Filter flags in C

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/export.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/python/export.py b/src/python/export.py
index 5e7fe53..739742f 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -29,7 +29,7 @@ import _location
 log = logging.getLogger("location.export")
 log.propagate = 1
 
-flags = {
+FLAGS = {
 	_location.NETWORK_FLAG_ANONYMOUS_PROXY    : "A1",
 	_location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
 	_location.NETWORK_FLAG_ANYCAST            : "A3",
@@ -186,12 +186,18 @@ class Exporter(object):
 
 			# Filter countries from special country codes
 			country_codes = [
-				country_code for country_code in countries if not country_code in flags.values()
+				country_code for country_code in countries if not country_code in FLAGS.values()
 			]
 
+			# Collect flags
+			flags = 0
+			for flag in FLAGS:
+				if FLAGS[flag] in countries:
+					flags |= flag
+
 			# Get all networks that match the family
 			networks = self.db.search_networks(family=family,
-				country_codes=country_codes, flatten=True)
+				country_codes=country_codes, flags=flags, flatten=True)
 
 			# Walk through all networks
 			for network in networks:
@@ -208,10 +214,10 @@ class Exporter(object):
 					pass
 
 				# Handle flags
-				for flag in flags:
+				for flag in FLAGS:
 					if network.has_flag(flag):
 						# Fetch the "fake" country code
-						country = flags[flag]
+						country = FLAGS[flag]
 
 						try:
 							writers[country].write(network)
-- 
2.20.1

From 84a2f0c2d9cbf8ae4225802c29ccba86561c77ed Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 17 Nov 2020 16:46:48 +0000
Subject: [PATCH 059/111] as: Add list for easier processing

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 Makefile.am           |   2 +
 src/as-list.c         | 138 ++++++++++++++++++++++++++++++++++++++++++
 src/database.c        |  29 +++++++--
 src/libloc.sym        |  15 ++++-
 src/loc/as-list.h     |  41 +++++++++++++
 src/loc/database.h    |   5 +-
 src/python/database.c |  58 ++++++++++++++++--
 src/python/export.py  |   2 +-
 8 files changed, 275 insertions(+), 15 deletions(-)
 create mode 100644 src/as-list.c
 create mode 100644 src/loc/as-list.h

diff --git a/Makefile.am b/Makefile.am
index f4ca3c8..d0cc793 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -91,6 +91,7 @@ EXTRA_DIST += \
 pkginclude_HEADERS = \
 	src/loc/libloc.h \
 	src/loc/as.h \
+	src/loc/as-list.h \
 	src/loc/compat.h \
 	src/loc/country.h \
 	src/loc/country-list.h \
@@ -109,6 +110,7 @@ lib_LTLIBRARIES = \
 src_libloc_la_SOURCES = \
 	src/libloc.c \
 	src/as.c \
+	src/as-list.c \
 	src/country.c \
 	src/country-list.c \
 	src/database.c \
diff --git a/src/as-list.c b/src/as-list.c
new file mode 100644
index 0000000..7c69eb0
--- /dev/null
+++ b/src/as-list.c
@@ -0,0 +1,138 @@
+/*
+	libloc - A library to determine the location of someone on the Internet
+
+	Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
+
+	This library is free software; you can redistribute it and/or
+	modify it under the terms of the GNU Lesser General Public
+	License as published by the Free Software Foundation; either
+	version 2.1 of the License, or (at your option) any later version.
+
+	This library is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+	Lesser General Public License for more details.
+*/
+
+#include <errno.h>
+#include <stdlib.h>
+
+#include <loc/as.h>
+#include <loc/as-list.h>
+#include <loc/private.h>
+
+struct loc_as_list {
+	struct loc_ctx* ctx;
+	int refcount;
+
+	struct loc_as* list[1024];
+	size_t size;
+	size_t max_size;
+};
+
+LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
+		struct loc_as_list** list) {
+	struct loc_as_list* l = calloc(1, sizeof(*l));
+	if (!l)
+		return -ENOMEM;
+
+	l->ctx = loc_ref(ctx);
+	l->refcount = 1;
+
+	// Do not allow this list to grow larger than this
+	l->max_size = 1024;
+
+	DEBUG(l->ctx, "AS list allocated at %p\n", l);
+	*list = l;
+
+	return 0;
+}
+
+LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
+	list->refcount++;
+
+	return list;
+}
+
+static void loc_as_list_free(struct loc_as_list* list) {
+	DEBUG(list->ctx, "Releasing AS list at %p\n", list);
+
+	loc_as_list_clear(list);
+
+	loc_unref(list->ctx);
+	free(list);
+}
+
+LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
+	if (!list)
+		return NULL;
+
+	if (--list->refcount > 0)
+		return list;
+
+	loc_as_list_free(list);
+	return NULL;
+}
+
+LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
+	return list->size;
+}
+
+LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
+	return list->size == 0;
+}
+
+LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
+	for (unsigned int i = 0; i < list->size; i++)
+		loc_as_unref(list->list[i]);
+}
+
+LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
+	// Check index
+	if (index >= list->size)
+		return NULL;
+
+	return loc_as_ref(list->list[index]);
+}
+
+LOC_EXPORT int loc_as_list_append(
+		struct loc_as_list* list, struct loc_as* as) {
+	if (loc_as_list_contains(list, as))
+		return 0;
+
+	// Check if we have space left
+	if (list->size == list->max_size) {
+		ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
+		return -ENOMEM;
+	}
+
+	DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
+
+	list->list[list->size++] = loc_as_ref(as);
+
+	return 0;
+}
+
+LOC_EXPORT int loc_as_list_contains(
+		struct loc_as_list* list, struct loc_as* as) {
+	for (unsigned int i = 0; i < list->size; i++) {
+		if (loc_as_cmp(as, list->list[i]) == 0)
+			return 1;
+	}
+
+	return 0;
+}
+
+LOC_EXPORT int loc_as_list_contains_number(
+		struct loc_as_list* list, uint32_t number) {
+	struct loc_as* as;
+
+	int r = loc_as_new(list->ctx, &as, number);
+	if (r)
+		return -1;
+
+	r = loc_as_list_contains(list, as);
+	loc_as_unref(as);
+
+	return r;
+}
diff --git a/src/database.c b/src/database.c
index 29823b2..51cb5cd 100644
--- a/src/database.c
+++ b/src/database.c
@@ -38,6 +38,7 @@
 
 #include <loc/libloc.h>
 #include <loc/as.h>
+#include <loc/as-list.h>
 #include <loc/compat.h>
 #include <loc/country.h>
 #include <loc/country-list.h>
@@ -101,7 +102,7 @@ struct loc_database_enumerator {
 	// Search string
 	char* string;
 	struct loc_country_list* countries;
-	uint32_t asn;
+	struct loc_as_list* asns;
 	enum loc_network_flags flags;
 	int family;
 
@@ -1036,9 +1037,20 @@ LOC_EXPORT int loc_database_enumerator_set_countries(
 	return 0;
 }
 
-LOC_EXPORT int loc_database_enumerator_set_asn(
-		struct loc_database_enumerator* enumerator, unsigned int asn) {
-	enumerator->asn = asn;
+LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
+		struct loc_database_enumerator* enumerator) {
+	if (!enumerator->asns)
+		return NULL;
+
+	return loc_as_list_ref(enumerator->asns);
+}
+
+LOC_EXPORT int loc_database_enumerator_set_asns(
+		struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
+	if (enumerator->asns)
+		loc_as_list_unref(enumerator->asns);
+
+	enumerator->asns = loc_as_list_ref(asns);
 
 	return 0;
 }
@@ -1123,6 +1135,12 @@ static int loc_network_match_countries(struct loc_network* network, struct loc_c
 	return loc_country_list_contains_code(countries, country_code);
 }
 
+static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
+	uint32_t asn = loc_network_get_asn(network);
+
+	return loc_as_list_contains_number(asns, asn);
+}
+
 static int loc_database_enumerator_filter_network(
 		struct loc_database_enumerator* enumerator, struct loc_network* network) {
 	// Skip if the family does not match
@@ -1134,8 +1152,7 @@ static int loc_database_enumerator_filter_network(
 		return 1;
 
 	// Skip if the ASN does not match
-	if (enumerator->asn &&
-			!loc_network_match_asn(network, enumerator->asn))
+	if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
 		return 1;
 
 	// Skip if flags do not match
diff --git a/src/libloc.sym b/src/libloc.sym
index 40e9f88..53273cd 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -37,6 +37,18 @@ global:
 	loc_as_set_name;
 	loc_as_unref;
 
+	# AS List
+	loc_as_list_append;
+	loc_as_list_clear;
+	loc_as_list_contains;
+	loc_as_list_contains_number;
+	loc_as_list_empty;
+	loc_as_list_get;
+	loc_as_list_new;
+	loc_as_list_ref;
+	loc_as_list_size;
+	loc_as_list_unref;
+
 	# Country
 	loc_country_cmp;
 	loc_country_code_is_valid;
@@ -78,13 +90,14 @@ global:
 	loc_database_verify;
 
 	# Database Enumerator
+	loc_database_enumerator_get_asns;
 	loc_database_enumerator_get_countries;
 	loc_database_enumerator_new;
 	loc_database_enumerator_next_as;
 	loc_database_enumerator_next_country;
 	loc_database_enumerator_next_network;
 	loc_database_enumerator_ref;
-	loc_database_enumerator_set_asn;
+	loc_database_enumerator_set_asns;
 	loc_database_enumerator_set_countries;
 	loc_database_enumerator_set_family;
 	loc_database_enumerator_set_flag;
diff --git a/src/loc/as-list.h b/src/loc/as-list.h
new file mode 100644
index 0000000..7b5c4e8
--- /dev/null
+++ b/src/loc/as-list.h
@@ -0,0 +1,41 @@
+/*
+	libloc - A library to determine the location of someone on the Internet
+
+	Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
+
+	This library is free software; you can redistribute it and/or
+	modify it under the terms of the GNU Lesser General Public
+	License as published by the Free Software Foundation; either
+	version 2.1 of the License, or (at your option) any later version.
+
+	This library is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+	Lesser General Public License for more details.
+*/
+
+#ifndef LIBLOC_AS_LIST_H
+#define LIBLOC_AS_LIST_H
+
+#include <loc/as.h>
+#include <loc/libloc.h>
+
+struct loc_as_list;
+
+int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
+struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
+struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
+
+size_t loc_as_list_size(struct loc_as_list* list);
+int loc_as_list_empty(struct loc_as_list* list);
+void loc_as_list_clear(struct loc_as_list* list);
+
+struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
+int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
+
+int loc_as_list_contains(
+	struct loc_as_list* list, struct loc_as* as);
+int loc_as_list_contains_number(
+	struct loc_as_list* list, uint32_t number);
+
+#endif
diff --git a/src/loc/database.h b/src/loc/database.h
index 246e5c5..70801f0 100644
--- a/src/loc/database.h
+++ b/src/loc/database.h
@@ -70,7 +70,10 @@ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerato
 struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
 int loc_database_enumerator_set_countries(
 	struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
-int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
+struct loc_as_list* loc_database_enumerator_get_asns(
+	struct loc_database_enumerator* enumerator);
+int loc_database_enumerator_set_asns(
+	struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
 int loc_database_enumerator_next_as(
diff --git a/src/python/database.c b/src/python/database.c
index e6f6f37..38a804c 100644
--- a/src/python/database.c
+++ b/src/python/database.c
@@ -17,6 +17,8 @@
 #include <Python.h>
 
 #include <loc/libloc.h>
+#include <loc/as.h>
+#include <loc/as-list.h>
 #include <loc/database.h>
 
 #include "locationmodule.h"
@@ -258,15 +260,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
 }
 
 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
-	char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
+	char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
 	PyObject* country_codes = NULL;
-	unsigned int asn = 0;
+	PyObject* asn_list = NULL;
 	int flags = 0;
 	int family = 0;
 	int flatten = 0;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
-			&PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
+			&PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
 		return NULL;
 
 	struct loc_database_enumerator* enumerator;
@@ -330,13 +332,57 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
 	}
 
 	// Set the ASN we are searching for
-	if (asn) {
-		r = loc_database_enumerator_set_asn(enumerator, asn);
+	if (asn_list) {
+		struct loc_as_list* asns;
+		r = loc_as_list_new(loc_ctx, &asns);
+		if (r) {
+			PyErr_SetString(PyExc_SystemError, "Could not create AS list");
+			return NULL;
+		}
+
+		for (unsigned int i = 0; i < PyList_Size(asn_list); i++) {
+			PyObject* item = PyList_GetItem(asn_list, i);
+
+			if (!PyLong_Check(item)) {
+				PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
+
+				loc_as_list_unref(asns);
+				return NULL;
+			}
+
+			unsigned long number = PyLong_AsLong(item);
 
+			struct loc_as* as;
+			r = loc_as_new(loc_ctx, &as, number);
+			if (r) {
+				PyErr_SetString(PyExc_SystemError, "Could not create AS");
+
+				loc_as_list_unref(asns);
+				loc_as_unref(as);
+				return NULL;
+			}
+
+			r = loc_as_list_append(asns, as);
+			if (r) {
+				PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
+
+				loc_as_list_unref(asns);
+				loc_as_unref(as);
+				return NULL;
+			}
+
+			loc_as_unref(as);
+		}
+
+		r = loc_database_enumerator_set_asns(enumerator, asns);
 		if (r) {
 			PyErr_SetFromErrno(PyExc_SystemError);
+
+			loc_as_list_unref(asns);
 			return NULL;
 		}
+
+		loc_as_list_unref(asns);
 	}
 
 	// Set the flags we are searching for
diff --git a/src/python/export.py b/src/python/export.py
index 739742f..f675eb3 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -197,7 +197,7 @@ class Exporter(object):
 
 			# Get all networks that match the family
 			networks = self.db.search_networks(family=family,
-				country_codes=country_codes, flags=flags, flatten=True)
+				country_codes=country_codes, asns=asns, flags=flags, flatten=True)
 
 			# Walk through all networks
 			for network in networks:
-- 
2.20.1

From 50120b991fc2fa4b7813096de87b42d700faf3e6 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 17 Nov 2020 16:56:43 +0000
Subject: [PATCH 060/111] database: Simplify network matching code

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/database.c b/src/database.c
index 51cb5cd..1a354f6 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
 	return 0;
 }
 
-static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
-	const char* country_code = loc_network_get_country_code(network);
-
-	return loc_country_list_contains_code(countries, country_code);
-}
-
 static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
 	uint32_t asn = loc_network_get_asn(network);
 
@@ -1148,8 +1142,14 @@ static int loc_database_enumerator_filter_network(
 		return 1;
 
 	// Skip if the country code does not match
-	if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
-		return 1;
+	if (enumerator->countries) {
+		if (!loc_country_list_empty(enumerator->countries)) {
+			const char* country_code = loc_network_get_country_code(network);
+
+			if (!loc_country_list_contains_code(enumerator->countries, country_code))
+				return 1;
+		}
+	}
 
 	// Skip if the ASN does not match
 	if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
-- 
2.20.1

From c1a36c943181da5cd2aef589a972d5027e529eb8 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 17 Nov 2020 16:58:55 +0000
Subject: [PATCH 061/111] database: Simplify AS matching code

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/database.c b/src/database.c
index 1a354f6..be93e00 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
 	return 0;
 }
 
-static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
-	uint32_t asn = loc_network_get_asn(network);
-
-	return loc_as_list_contains_number(asns, asn);
-}
-
 static int loc_database_enumerator_filter_network(
 		struct loc_database_enumerator* enumerator, struct loc_network* network) {
 	// Skip if the family does not match
@@ -1152,8 +1146,14 @@ static int loc_database_enumerator_filter_network(
 	}
 
 	// Skip if the ASN does not match
-	if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
-		return 1;
+	if (enumerator->asns) {
+		if (!loc_as_list_empty(enumerator->asns)) {
+			uint32_t asn = loc_network_get_asn(network);
+
+			if (!loc_as_list_contains_number(enumerator->asns, asn))
+				return 1;
+		}
+	}
 
 	// Skip if flags do not match
 	if (enumerator->flags &&
-- 
2.20.1

From d5205091f9cc1ff987e483325d48696459df08d8 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 17 Nov 2020 17:50:17 +0000
Subject: [PATCH 062/111] countries: Make list grow dynamically

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/country-list.c | 38 ++++++++++++++++++++++++++------------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/src/country-list.c b/src/country-list.c
index ae0d71a..1ce2d06 100644
--- a/src/country-list.c
+++ b/src/country-list.c
@@ -25,11 +25,27 @@ struct loc_country_list {
 	struct loc_ctx* ctx;
 	int refcount;
 
-	struct loc_country* list[1024];
+	struct loc_country** elements;
+	size_t elements_size;
+
 	size_t size;
-	size_t max_size;
 };
 
+static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
+	DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
+		list, size, list->elements_size + size);
+
+	struct loc_country** elements = reallocarray(list->elements,
+			list->elements_size + size, sizeof(*list->elements));
+	if (!elements)
+		return -errno;
+
+	list->elements = elements;
+	list->elements_size += size;
+
+	return 0;
+}
+
 LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
 		struct loc_country_list** list) {
 	struct loc_country_list* l = calloc(1, sizeof(*l));
@@ -39,9 +55,6 @@ LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
 	l->ctx = loc_ref(ctx);
 	l->refcount = 1;
 
-	// Do not allow this list to grow larger than this
-	l->max_size = 1024;
-
 	DEBUG(l->ctx, "Country list allocated at %p\n", l);
 	*list = l;
 
@@ -84,7 +97,7 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
 
 LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
 	for (unsigned int i = 0; i < list->size; i++)
-		loc_country_unref(list->list[i]);
+		loc_country_unref(list->elements[i]);
 }
 
 LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* lis
 	if (index >= list->size)
 		return NULL;
 
-	return loc_country_ref(list->list[index]);
+	return loc_country_ref(list->elements[index]);
 }
 
 LOC_EXPORT int loc_country_list_append(
@@ -101,14 +114,15 @@ LOC_EXPORT int loc_country_list_append(
 		return 0;
 
 	// Check if we have space left
-	if (list->size == list->max_size) {
-		ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
-		return -ENOMEM;
+	if (list->size >= list->elements_size) {
+		int r = loc_country_list_grow(list, 64);
+		if (r)
+			return r;
 	}
 
 	DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
 
-	list->list[list->size++] = loc_country_ref(country);
+	list->elements[list->size++] = loc_country_ref(country);
 
 	return 0;
 }
@@ -116,7 +130,7 @@ LOC_EXPORT int loc_country_list_append(
 LOC_EXPORT int loc_country_list_contains(
 		struct loc_country_list* list, struct loc_country* country) {
 	for (unsigned int i = 0; i < list->size; i++) {
-		if (loc_country_cmp(country, list->list[i]) == 0)
+		if (loc_country_cmp(country, list->elements[i]) == 0)
 			return 1;
 	}
 
-- 
2.20.1

From 3b44e4211371d2103f89ba8f056b15edb7778fac Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 17 Nov 2020 17:55:51 +0000
Subject: [PATCH 063/111] networks: Make list grow dynamically

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network-list.c | 60 ++++++++++++++++++++++++++++------------------
 1 file changed, 37 insertions(+), 23 deletions(-)

diff --git a/src/network-list.c b/src/network-list.c
index 1f6e80e..4912c02 100644
--- a/src/network-list.c
+++ b/src/network-list.c
@@ -25,11 +25,27 @@ struct loc_network_list {
 	struct loc_ctx* ctx;
 	int refcount;
 
-	struct loc_network* list[1024];
+	struct loc_network** elements;
+	size_t elements_size;
+
 	size_t size;
-	size_t max_size;
 };
 
+static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
+	DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
+		list, size, list->elements_size + size);
+
+	struct loc_network** elements = reallocarray(list->elements,
+			list->elements_size + size, sizeof(*list->elements));
+	if (!elements)
+		return -errno;
+
+	list->elements = elements;
+	list->elements_size += size;
+
+	return 0;
+}
+
 LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
 		struct loc_network_list** list) {
 	struct loc_network_list* l = calloc(1, sizeof(*l));
@@ -39,9 +55,6 @@ LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
 	l->ctx = loc_ref(ctx);
 	l->refcount = 1;
 
-	// Do not allow this list to grow larger than this
-	l->max_size = 1024;
-
 	DEBUG(l->ctx, "Network list allocated at %p\n", l);
 	*list = l;
 	return 0;
@@ -57,7 +70,7 @@ static void loc_network_list_free(struct loc_network_list* list) {
 	DEBUG(list->ctx, "Releasing network list at %p\n", list);
 
 	for (unsigned int i = 0; i < list->size; i++)
-		loc_network_unref(list->list[i]);
+		loc_network_unref(list->elements[i]);
 
 	loc_unref(list->ctx);
 	free(list);
@@ -84,7 +97,7 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
 
 LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
 	for (unsigned int i = 0; i < list->size; i++)
-		loc_network_unref(list->list[i]);
+		loc_network_unref(list->elements[i]);
 
 	list->size = 0;
 }
@@ -94,7 +107,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
 	char* s;
 
 	for (unsigned int i = 0; i < list->size; i++) {
-		network = list->list[i];
+		network = list->elements[i];
 
 		s = loc_network_str(network);
 
@@ -108,7 +121,7 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
 	if (index >= list->size)
 		return NULL;
 
-	return loc_network_ref(list->list[index]);
+	return loc_network_ref(list->elements[index]);
 }
 
 LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
@@ -117,14 +130,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
 		return 0;
 
 	// Check if we have space left
-	if (list->size == list->max_size) {
-		ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
-		return -ENOMEM;
+	if (list->size >= list->elements_size) {
+		int r = loc_network_list_grow(list, 64);
+		if (r)
+			return r;
 	}
 
 	DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
 
-	list->list[list->size++] = loc_network_ref(network);
+	list->elements[list->size++] = loc_network_ref(network);
 
 	return 0;
 }
@@ -136,7 +150,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
 		return NULL;
 	}
 
-	struct loc_network* network = list->list[--list->size];
+	struct loc_network* network = list->elements[--list->size];
 
 	DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
 
@@ -150,11 +164,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
 		return NULL;
 	}
 
-	struct loc_network* network = list->list[0];
+	struct loc_network* network = list->elements[0];
 
 	// Move all elements to the top of the stack
 	for (unsigned int i = 0; i < --list->size; i++) {
-		list->list[i] = list->list[i+1];
+		list->elements[i] = list->elements[i+1];
 	}
 
 	DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
@@ -164,7 +178,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
 
 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
 	for (unsigned int i = 0; i < list->size; i++) {
-		if (loc_network_eq(list->list[i], network))
+		if (loc_network_eq(list->elements[i], network))
 			return 1;
 	}
 
@@ -176,11 +190,11 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
 	if (i1 >= list->size || i2 >= list->size)
 		return;
 
-	struct loc_network* network1 = list->list[i1];
-	struct loc_network* network2 = list->list[i2];
+	struct loc_network* network1 = list->elements[i1];
+	struct loc_network* network2 = list->elements[i2];
 
-	list->list[i1] = network2;
-	list->list[i2] = network1;
+	list->elements[i1] = network2;
+	list->elements[i2] = network1;
 }
 
 LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
@@ -200,7 +214,7 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
 		swapped = 0;
 
 		for (unsigned int i = 1; i < n; i++) {
-			if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
+			if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) {
 				loc_network_list_swap(list, i-1, i);
 				swapped = 1;
 			}
@@ -215,7 +229,7 @@ LOC_EXPORT int loc_network_list_merge(
 	int r;
 
 	for (unsigned int i = 0; i < other->size; i++) {
-		r = loc_network_list_push(self, other->list[i]);
+		r = loc_network_list_push(self, other->elements[i]);
 		if (r)
 			return r;
 	}
-- 
2.20.1

From 1a415f8c555f4fe9a68eb2a897c4a1fc0d33db25 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 17 Nov 2020 17:57:55 +0000
Subject: [PATCH 064/111] as: Make lists grow dynamically

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/as-list.c | 38 ++++++++++++++++++++++++++------------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/src/as-list.c b/src/as-list.c
index 7c69eb0..17de23e 100644
--- a/src/as-list.c
+++ b/src/as-list.c
@@ -25,11 +25,27 @@ struct loc_as_list {
 	struct loc_ctx* ctx;
 	int refcount;
 
-	struct loc_as* list[1024];
+	struct loc_as** elements;
+	size_t elements_size;
+
 	size_t size;
-	size_t max_size;
 };
 
+static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
+	DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
+		list, size, list->elements_size + size);
+
+	struct loc_as** elements = reallocarray(list->elements,
+			list->elements_size + size, sizeof(*list->elements));
+	if (!elements)
+		return -errno;
+
+	list->elements = elements;
+	list->elements_size += size;
+
+	return 0;
+}
+
 LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
 		struct loc_as_list** list) {
 	struct loc_as_list* l = calloc(1, sizeof(*l));
@@ -39,9 +55,6 @@ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
 	l->ctx = loc_ref(ctx);
 	l->refcount = 1;
 
-	// Do not allow this list to grow larger than this
-	l->max_size = 1024;
-
 	DEBUG(l->ctx, "AS list allocated at %p\n", l);
 	*list = l;
 
@@ -84,7 +97,7 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
 
 LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
 	for (unsigned int i = 0; i < list->size; i++)
-		loc_as_unref(list->list[i]);
+		loc_as_unref(list->elements[i]);
 }
 
 LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index
 	if (index >= list->size)
 		return NULL;
 
-	return loc_as_ref(list->list[index]);
+	return loc_as_ref(list->elements[index]);
 }
 
 LOC_EXPORT int loc_as_list_append(
@@ -101,14 +114,15 @@ LOC_EXPORT int loc_as_list_append(
 		return 0;
 
 	// Check if we have space left
-	if (list->size == list->max_size) {
-		ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
-		return -ENOMEM;
+	if (list->size >= list->elements_size) {
+		int r = loc_as_list_grow(list, 64);
+		if (r)
+			return r;
 	}
 
 	DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
 
-	list->list[list->size++] = loc_as_ref(as);
+	list->elements[list->size++] = loc_as_ref(as);
 
 	return 0;
 }
@@ -116,7 +130,7 @@ LOC_EXPORT int loc_as_list_append(
 LOC_EXPORT int loc_as_list_contains(
 		struct loc_as_list* list, struct loc_as* as) {
 	for (unsigned int i = 0; i < list->size; i++) {
-		if (loc_as_cmp(as, list->list[i]) == 0)
+		if (loc_as_cmp(as, list->elements[i]) == 0)
 			return 1;
 	}
 
-- 
2.20.1

From e6592434ee7836507c1f436ec3b0db3bc81a81b9 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 17 Nov 2020 18:13:49 +0000
Subject: [PATCH 065/111] export: Change back to use Network objects

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/export.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/python/export.py b/src/python/export.py
index f675eb3..67e437f 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -144,9 +144,7 @@ class XTGeoIPOutputWriter(OutputWriter):
 	def _write_network(self, network):
 		for address in (network.first_address, network.last_address):
 			# Convert this into a string of bits
-			bytes = socket.inet_pton(
-				socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
-			)
+			bytes = socket.inet_pton(network.family, address)
 
 			self.f.write(bytes)
 
-- 
2.20.1

From 248f5e0419f2349253b8ea96e477c15649fe2173 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 17 Nov 2020 18:14:15 +0000
Subject: [PATCH 066/111] Actually clear all lists

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/as-list.c      | 8 ++++++++
 src/country-list.c | 8 ++++++++
 src/network-list.c | 6 ++++++
 3 files changed, 22 insertions(+)

diff --git a/src/as-list.c b/src/as-list.c
index 17de23e..76620c7 100644
--- a/src/as-list.c
+++ b/src/as-list.c
@@ -96,8 +96,16 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
 }
 
 LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
+	if (!list->elements)
+		return;
+
 	for (unsigned int i = 0; i < list->size; i++)
 		loc_as_unref(list->elements[i]);
+
+	free(list->elements);
+	list->elements_size = 0;
+
+	list->size = 0;
 }
 
 LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
diff --git a/src/country-list.c b/src/country-list.c
index 1ce2d06..1c49c47 100644
--- a/src/country-list.c
+++ b/src/country-list.c
@@ -96,8 +96,16 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
 }
 
 LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
+	if (!list->elements)
+		return;
+
 	for (unsigned int i = 0; i < list->size; i++)
 		loc_country_unref(list->elements[i]);
+
+	free(list->elements);
+	list->elements_size = 0;
+
+	list->size = 0;
 }
 
 LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
diff --git a/src/network-list.c b/src/network-list.c
index 4912c02..9cb4547 100644
--- a/src/network-list.c
+++ b/src/network-list.c
@@ -96,9 +96,15 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
 }
 
 LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
+	if (!list->elements)
+		return;
+
 	for (unsigned int i = 0; i < list->size; i++)
 		loc_network_unref(list->elements[i]);
 
+	free(list->elements);
+	list->elements_size = 0;
+
 	list->size = 0;
 }
 
-- 
2.20.1

From c98ebf8aae2aa141193db52cd9429b1ded5b09c4 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 17 Nov 2020 18:34:51 +0000
Subject: [PATCH 067/111] database: Do not clean up python list

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/database.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/python/database.c b/src/python/database.c
index 38a804c..ed22275 100644
--- a/src/python/database.c
+++ b/src/python/database.c
@@ -325,9 +325,14 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
 			loc_country_unref(country);
 		}
 
-		loc_database_enumerator_set_countries(enumerator, countries);
+		r = loc_database_enumerator_set_countries(enumerator, countries);
+		if (r) {
+			PyErr_SetFromErrno(PyExc_SystemError);
+
+			loc_as_list_unref(countries);
+			return NULL;
+		}
 
-		Py_DECREF(country_codes);
 		loc_country_list_unref(countries);
 	}
 
-- 
2.20.1

From 5470d06cb59027f4e04b6d576763dbf7f1093fde Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 17 Nov 2020 19:01:04 +0000
Subject: [PATCH 068/111] database: Free filter lists in enumerator

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/database.c b/src/database.c
index be93e00..ca35fe1 100644
--- a/src/database.c
+++ b/src/database.c
@@ -950,6 +950,12 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera
 	if (enumerator->string)
 		free(enumerator->string);
 
+	if (enumerator->countries)
+		loc_country_list_unref(enumerator->countries);
+
+	if (enumerator->asns)
+		loc_as_list_unref(enumerator->asns);
+
 	// Free network search
 	free(enumerator->networks_visited);
 
-- 
2.20.1

From e0e96878d3df51c4a265d51d088005dedf9335e3 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 18 Nov 2020 13:18:52 +0000
Subject: [PATCH 069/111] database: Add debug output to filtering

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/src/database.c b/src/database.c
index ca35fe1..83dd752 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1138,33 +1138,36 @@ static int loc_database_enumerator_stack_push_node(
 static int loc_database_enumerator_filter_network(
 		struct loc_database_enumerator* enumerator, struct loc_network* network) {
 	// Skip if the family does not match
-	if (enumerator->family && loc_network_address_family(network) != enumerator->family)
+	if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
+		DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
 		return 1;
+	}
 
 	// Skip if the country code does not match
-	if (enumerator->countries) {
-		if (!loc_country_list_empty(enumerator->countries)) {
-			const char* country_code = loc_network_get_country_code(network);
+	if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
+		const char* country_code = loc_network_get_country_code(network);
 
-			if (!loc_country_list_contains_code(enumerator->countries, country_code))
-				return 1;
+		if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
+			DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
+			return 1;
 		}
 	}
 
 	// Skip if the ASN does not match
-	if (enumerator->asns) {
-		if (!loc_as_list_empty(enumerator->asns)) {
-			uint32_t asn = loc_network_get_asn(network);
+	if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
+		uint32_t asn = loc_network_get_asn(network);
 
-			if (!loc_as_list_contains_number(enumerator->asns, asn))
-				return 1;
+		if (!loc_as_list_contains_number(enumerator->asns, asn)) {
+			DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
+			return 1;
 		}
 	}
 
 	// Skip if flags do not match
-	if (enumerator->flags &&
-			!loc_network_match_flag(network, enumerator->flags))
+	if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
+		DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
 		return 1;
+	}
 
 	// Do not filter
 	return 0;
-- 
2.20.1

From bce0c9295ff8ff9488f24babe01ce851228d0b1e Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 18 Nov 2020 13:19:04 +0000
Subject: [PATCH 070/111] export: Remove filtering for flags

The filter is an AND filter and if we set the flags from
the special country codes, we won't get back much.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/export.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/python/export.py b/src/python/export.py
index 67e437f..4219957 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -187,15 +187,9 @@ class Exporter(object):
 				country_code for country_code in countries if not country_code in FLAGS.values()
 			]
 
-			# Collect flags
-			flags = 0
-			for flag in FLAGS:
-				if FLAGS[flag] in countries:
-					flags |= flag
-
 			# Get all networks that match the family
 			networks = self.db.search_networks(family=family,
-				country_codes=country_codes, asns=asns, flags=flags, flatten=True)
+				country_codes=country_codes, asns=asns, flatten=True)
 
 			# Walk through all networks
 			for network in networks:
-- 
2.20.1

From 627bf1daaae1510cfd4016297ed16b82df209aae Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 18 Nov 2020 13:33:45 +0000
Subject: [PATCH 071/111] python: Remove unnecessary db object from writers

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/export.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/python/export.py b/src/python/export.py
index 4219957..5bc9f30 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -39,8 +39,8 @@ class OutputWriter(object):
 	suffix = "networks"
 	mode = "w"
 
-	def __init__(self, db, f, prefix=None, flatten=True):
-		self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten
+	def __init__(self, f, prefix=None, flatten=True):
+		self.f, self.prefix, self.flatten = f, prefix, flatten
 
 		# The previously written network
 		self._last_network = None
@@ -49,13 +49,13 @@ class OutputWriter(object):
 		self._write_header()
 
 	@classmethod
-	def open(cls, db, filename, **kwargs):
+	def open(cls, filename, **kwargs):
 		"""
 			Convenience function to open a file
 		"""
 		f = open(filename, cls.mode)
 
-		return cls(db, f, **kwargs)
+		return cls(f, **kwargs)
 
 	def __repr__(self):
 		return "<%s f=%s>" % (self.__class__.__name__, self.f)
@@ -172,7 +172,7 @@ class Exporter(object):
 					directory, prefix=country_code, suffix=self.writer.suffix, family=family,
 				)
 
-				writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code)
+				writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
 
 			# Create writers for ASNs
 			for asn in asns:
@@ -180,7 +180,7 @@ class Exporter(object):
 					directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
 				)
 
-				writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
+				writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
 
 			# Filter countries from special country codes
 			country_codes = [
-- 
2.20.1

From 9cb56ac9adafafa6e452009c2fa2d42e94474e11 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 18 Nov 2020 13:34:50 +0000
Subject: [PATCH 072/111] location: End lookup after an invalid IP address was
 passed

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/location.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/python/location.in b/src/python/location.in
index 0d09210..6885ea0 100644
--- a/src/python/location.in
+++ b/src/python/location.in
@@ -253,6 +253,7 @@ class CLI(object):
 				network = db.lookup(address)
 			except ValueError:
 				print(_("Invalid IP address: %s") % address, file=sys.stderr)
+				return 2
 
 			args = {
 				"address" : address,
-- 
2.20.1

From 2a550d12208f8bc8002e05ac08613312df26b20e Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 19 Nov 2020 12:03:33 +0000
Subject: [PATCH 073/111] python: Fix download of database

This was all messed up in 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1
when the change of type for the timestamp wasn't changed everywhere
else.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/downloader.py | 6 +++---
 src/python/location.in   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/python/downloader.py b/src/python/downloader.py
index 87bbb68..05f7872 100644
--- a/src/python/downloader.py
+++ b/src/python/downloader.py
@@ -119,8 +119,8 @@ class Downloader(object):
 
 		headers = {}
 		if timestamp:
-			headers["If-Modified-Since"] = timestamp.strftime(
-				"%a, %d %b %Y %H:%M:%S GMT",
+			headers["If-Modified-Since"] = time.strftime(
+				"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
 			)
 
 		t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
@@ -195,7 +195,7 @@ class Downloader(object):
 		db = Database(f.name)
 
 		# Database is not recent
-		if timestamp and db.created_at < timestamp.timestamp():
+		if timestamp and db.created_at < timestamp:
 			return False
 
 		log.info("Downloaded new database from %s" % (time.strftime(
diff --git a/src/python/location.in b/src/python/location.in
index 6885ea0..b30beae 100644
--- a/src/python/location.in
+++ b/src/python/location.in
@@ -433,7 +433,7 @@ class CLI(object):
 
 		# Try downloading a new database
 		try:
-			t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
+			t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
 
 		# If no file could be downloaded, log a message
 		except FileNotFoundError as e:
-- 
2.20.1

From a1a00053300cff3c0f690d52377c76c83c2a08b2 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 19 Nov 2020 12:34:11 +0000
Subject: [PATCH 074/111] python: Add property to return IP addresses as bytes

This avoids calling inet_pton to parse IP addresses from string

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym        |  2 ++
 src/loc/network.h     |  2 ++
 src/network.c         |  8 ++++++++
 src/python/database.c |  2 +-
 src/python/export.py  |  7 ++-----
 src/python/network.c  | 40 ++++++++++++++++++++++++++++++++++++++++
 6 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/src/libloc.sym b/src/libloc.sym
index 53273cd..406dd15 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -113,6 +113,8 @@ global:
 	loc_network_format_last_address;
 	loc_network_get_asn;
 	loc_network_get_country_code;
+	loc_network_get_first_address;
+	loc_network_get_last_address;
 	loc_network_gt;
 	loc_network_has_flag;
 	loc_network_is_subnet;
diff --git a/src/loc/network.h b/src/loc/network.h
index d86b685..4b7410c 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -39,7 +39,9 @@ struct loc_network* loc_network_unref(struct loc_network* network);
 char* loc_network_str(struct loc_network* network);
 int loc_network_address_family(struct loc_network* network);
 
+const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
 char* loc_network_format_first_address(struct loc_network* network);
+const struct in6_addr* loc_network_get_last_address(struct loc_network* network);
 char* loc_network_format_last_address(struct loc_network* network);
 int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
 
diff --git a/src/network.c b/src/network.c
index 28ca2df..4c8787a 100644
--- a/src/network.c
+++ b/src/network.c
@@ -343,10 +343,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc
 	return string;
 }
 
+LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) {
+	return &network->first_address;
+}
+
 LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) {
 	return loc_network_format_address(network, &network->first_address);
 }
 
+LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) {
+	return &network->last_address;
+}
+
 LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
 	return loc_network_format_address(network, &network->last_address);
 }
diff --git a/src/python/database.c b/src/python/database.c
index ed22275..f385c61 100644
--- a/src/python/database.c
+++ b/src/python/database.c
@@ -329,7 +329,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
 		if (r) {
 			PyErr_SetFromErrno(PyExc_SystemError);
 
-			loc_as_list_unref(countries);
+			loc_country_list_unref(countries);
 			return NULL;
 		}
 
diff --git a/src/python/export.py b/src/python/export.py
index 5bc9f30..6b39878 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -142,11 +142,8 @@ class XTGeoIPOutputWriter(OutputWriter):
 	mode = "wb"
 
 	def _write_network(self, network):
-		for address in (network.first_address, network.last_address):
-			# Convert this into a string of bits
-			bytes = socket.inet_pton(network.family, address)
-
-			self.f.write(bytes)
+		for address in (network._first_address, network._last_address):
+			self.f.write(address)
 
 
 formats = {
diff --git a/src/python/network.c b/src/python/network.c
index ed91d65..742b472 100644
--- a/src/python/network.c
+++ b/src/python/network.c
@@ -215,6 +215,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) {
 	return obj;
 }
 
+static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) {
+	struct in_addr address4;
+
+	// Convert IPv4 addresses to struct in_addr
+	if (IN6_IS_ADDR_V4MAPPED(address6)) {
+		address4.s_addr = address6->s6_addr32[3];
+
+		return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4));
+	}
+
+	// Return IPv6 addresses as they are
+	return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6));
+}
+
+static PyObject* Network_get__first_address(NetworkObject* self) {
+	const struct in6_addr* address = loc_network_get_first_address(self->network);
+
+	return PyBytes_FromAddress(address);
+}
+
 static PyObject* Network_get_last_address(NetworkObject* self) {
 	char* address = loc_network_format_last_address(self->network);
 
@@ -224,6 +244,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
 	return obj;
 }
 
+static PyObject* Network_get__last_address(NetworkObject* self) {
+	const struct in6_addr* address = loc_network_get_last_address(self->network);
+
+	return PyBytes_FromAddress(address);
+}
+
 static struct PyMethodDef Network_methods[] = {
 	{
 		"exclude",
@@ -281,6 +307,13 @@ static struct PyGetSetDef Network_getsetters[] = {
 		NULL,
 		NULL,
 	},
+	{
+		"_first_address",
+		(getter)Network_get__first_address,
+		NULL,
+		NULL,
+		NULL,
+	},
 	{
 		"last_address",
 		(getter)Network_get_last_address,
@@ -288,6 +321,13 @@ static struct PyGetSetDef Network_getsetters[] = {
 		NULL,
 		NULL,
 	},
+	{
+		"_last_address",
+		(getter)Network_get__last_address,
+		NULL,
+		NULL,
+		NULL,
+	},
 	{ NULL },
 };
 
-- 
2.20.1

From 90d2194a876c223f9124ce9e27bdee6a6b49ff6a Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 19 Nov 2020 12:40:01 +0000
Subject: [PATCH 075/111] export: Remove old flattening feature

The database enumerator now only returns networks that will
never overlap.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/export.py | 34 ++++++----------------------------
 1 file changed, 6 insertions(+), 28 deletions(-)

diff --git a/src/python/export.py b/src/python/export.py
index 6b39878..4702bcf 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -39,11 +39,8 @@ class OutputWriter(object):
 	suffix = "networks"
 	mode = "w"
 
-	def __init__(self, f, prefix=None, flatten=True):
-		self.f, self.prefix, self.flatten = f, prefix, flatten
-
-		# The previously written network
-		self._last_network = None
+	def __init__(self, f, prefix=None):
+		self.f, self.prefix = f, prefix
 
 		# Immediately write the header
 		self._write_header()
@@ -60,18 +57,6 @@ class OutputWriter(object):
 	def __repr__(self):
 		return "<%s f=%s>" % (self.__class__.__name__, self.f)
 
-	def _flatten(self, network):
-		"""
-			Checks if the given network needs to be written to file,
-			or if it is a subnet of the previously written network.
-		"""
-		if self._last_network and network.is_subnet_of(self._last_network):
-			return True
-
-		# Remember this network for the next call
-		self._last_network = network
-		return False
-
 	def _write_header(self):
 		"""
 			The header of the file
@@ -84,15 +69,8 @@ class OutputWriter(object):
 		"""
 		pass
 
-	def _write_network(self, network):
-		self.f.write("%s\n" % network)
-
 	def write(self, network):
-		if self.flatten and self._flatten(network):
-			log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
-			return
-
-		return self._write_network(network)
+		self.f.write("%s\n" % network)
 
 	def finish(self):
 		"""
@@ -113,7 +91,7 @@ class IpsetOutputWriter(OutputWriter):
 	def _write_header(self):
 		self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
 
-	def _write_network(self, network):
+	def write(self, network):
 		self.f.write("add %s %s\n" % (self.prefix, network))
 
 
@@ -129,7 +107,7 @@ class NftablesOutputWriter(OutputWriter):
 	def _write_footer(self):
 		self.f.write("}\n")
 
-	def _write_network(self, network):
+	def write(self, network):
 		self.f.write("	%s,\n" % network)
 
 
@@ -141,7 +119,7 @@ class XTGeoIPOutputWriter(OutputWriter):
 	suffix = "iv"
 	mode = "wb"
 
-	def _write_network(self, network):
+	def write(self, network):
 		for address in (network._first_address, network._last_address):
 			self.f.write(address)
 
-- 
2.20.1

From 90188dad86223380b0854b523b63ec024117c5f2 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 19 Nov 2020 12:41:19 +0000
Subject: [PATCH 076/111] export: Speed-up export in xt_geoip format

Removing the loop avoids creating a tuple and then iterating over it

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/python/export.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/python/export.py b/src/python/export.py
index 4702bcf..f0eae26 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -120,8 +120,8 @@ class XTGeoIPOutputWriter(OutputWriter):
 	mode = "wb"
 
 	def write(self, network):
-		for address in (network._first_address, network._last_address):
-			self.f.write(address)
+		self.f.write(network._first_address)
+		self.f.write(network._last_address)
 
 
 formats = {
-- 
2.20.1

From 6661692f3bc8f788af8b75ae25561f4cc4a2acb5 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 19 Nov 2020 12:48:46 +0000
Subject: [PATCH 077/111] database: Disable some useless code when not running
 in debug mode

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/database.c b/src/database.c
index 83dd752..ef4f505 100644
--- a/src/database.c
+++ b/src/database.c
@@ -619,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
 	}
 
 	clock_t end = clock();
-	DEBUG(db->ctx, "Signature checked in %.4fms\n",
+	INFO(db->ctx, "Signature checked in %.4fms\n",
 		(double)(end - start) / CLOCKS_PER_SEC * 1000);
 
 CLEANUP:
@@ -679,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
 	off_t lo = 0;
 	off_t hi = db->as_count - 1;
 
+#ifdef ENABLE_DEBUG
 	// Save start time
 	clock_t start = clock();
+#endif
 
 	while (lo <= hi) {
 		off_t i = (lo + hi) / 2;
@@ -693,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
 		// Check if this is a match
 		uint32_t as_number = loc_as_get_number(*as);
 		if (as_number == number) {
+#ifdef ENABLE_DEBUG
 			clock_t end = clock();
 
 			// Log how fast this has been
 			DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
 				(double)(end - start) / CLOCKS_PER_SEC * 1000);
+#endif
 
 			return 0;
 		}
@@ -741,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
 			return -1;
 	}
 
+#ifdef ENABLE_DEBUG
 	if (r == 0) {
 		char* string = loc_network_str(*network);
 		DEBUG(db->ctx, "Got network %s\n", string);
 		free(string);
 	}
+#endif
 
 	return r;
 }
@@ -840,17 +846,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
 
 	*network = NULL;
 
+#ifdef ENABLE_DEBUG
 	// Save start time
 	clock_t start = clock();
+#endif
 
 	int r = __loc_database_lookup(db, address, network, &network_address,
 		db->network_nodes_v1, 0);
 
+#ifdef ENABLE_DEBUG
 	clock_t end = clock();
 
 	// Log how fast this has been
 	DEBUG(db->ctx, "Executed network search in %.4fms\n",
 		(double)(end - start) / CLOCKS_PER_SEC * 1000);
+#endif
 
 	return r;
 }
@@ -897,8 +907,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
 	off_t lo = 0;
 	off_t hi = db->countries_count - 1;
 
+#ifdef ENABLE_DEBUG
 	// Save start time
 	clock_t start = clock();
+#endif
 
 	while (lo <= hi) {
 		off_t i = (lo + hi) / 2;
@@ -913,11 +925,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
 		int result = strcmp(code, cc);
 
 		if (result == 0) {
+#ifdef ENABLE_DEBUG
 			clock_t end = clock();
 
 			// Log how fast this has been
 			DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
 				(double)(end - start) / CLOCKS_PER_SEC * 1000);
+#endif
 
 			return 0;
 		}
-- 
2.20.1

From a17c353bce55a9f38f67b4b7d2425194cfa208e7 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 20 Nov 2020 14:43:31 +0000
Subject: [PATCH 078/111] test: Add tests to network-lists

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 Makefile.am             |  10 ++++
 src/.gitignore          |   1 +
 src/test-network-list.c | 126 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 137 insertions(+)
 create mode 100644 src/test-network-list.c

diff --git a/Makefile.am b/Makefile.am
index d0cc793..ebd7e17 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -318,6 +318,7 @@ check_PROGRAMS = \
 	src/test-database \
 	src/test-as \
 	src/test-network \
+	src/test-network-list \
 	src/test-country \
 	src/test-signature
 
@@ -357,6 +358,15 @@ src_test_network_CFLAGS = \
 src_test_network_LDADD = \
 	src/libloc.la
 
+src_test_network_list_SOURCES = \
+	src/test-network-list.c
+
+src_test_network_list_CFLAGS = \
+	$(TESTS_CFLAGS)
+
+src_test_network_list_LDADD = \
+	src/libloc.la
+
 src_test_stringpool_SOURCES = \
 	src/test-stringpool.c
 
diff --git a/src/.gitignore b/src/.gitignore
index caf80b5..3ccbdb8 100644
--- a/src/.gitignore
+++ b/src/.gitignore
@@ -10,5 +10,6 @@ test-libloc
 test-database
 test-country
 test-network
+test-network-list
 test-signature
 test-stringpool
diff --git a/src/test-network-list.c b/src/test-network-list.c
new file mode 100644
index 0000000..3061d63
--- /dev/null
+++ b/src/test-network-list.c
@@ -0,0 +1,126 @@
+/*
+	libloc - A library to determine the location of someone on the Internet
+
+	Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
+
+	This program is free software; you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation; either version 2 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+*/
+
+#include <errno.h>
+#include <stdio.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include <syslog.h>
+
+#include <loc/libloc.h>
+#include <loc/network.h>
+#include <loc/network-list.h>
+
+int main(int argc, char** argv) {
+	int err;
+
+	struct loc_ctx* ctx;
+	err = loc_new(&ctx);
+	if (err < 0)
+		exit(EXIT_FAILURE);
+
+	// Enable debug logging
+	loc_set_log_priority(ctx, LOG_DEBUG);
+
+	// Create a network
+	struct loc_network* network1;
+	err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32");
+	if (err) {
+		fprintf(stderr, "Could not create the network1\n");
+		exit(EXIT_FAILURE);
+	}
+
+	struct loc_network* subnet1;
+	err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48");
+	if (err) {
+		fprintf(stderr, "Could not create the subnet1\n");
+		exit(EXIT_FAILURE);
+	}
+
+	struct loc_network* subnet2;
+	err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48");
+	if (err) {
+		fprintf(stderr, "Could not create the subnet2\n");
+		exit(EXIT_FAILURE);
+	}
+
+	// Make a list with both subnets
+	struct loc_network_list* subnets;
+	err = loc_network_list_new(ctx, &subnets);
+	if (err) {
+		fprintf(stderr, "Could not create subnets list\n");
+		exit(EXIT_FAILURE);
+	}
+
+	size_t size = loc_network_list_size(subnets);
+	if (size > 0) {
+		fprintf(stderr, "The list is not empty: %zu\n", size);
+		exit(EXIT_FAILURE);
+	}
+
+	err = loc_network_list_push(subnets, subnet1);
+	if (err) {
+		fprintf(stderr, "Could not add subnet1 to subnets list\n");
+		exit(EXIT_FAILURE);
+	}
+
+	if (loc_network_list_empty(subnets)) {
+		fprintf(stderr, "The subnets list reports that it is empty\n");
+		exit(EXIT_FAILURE);
+	}
+
+	err = loc_network_list_push(subnets, subnet2);
+	if (err) {
+		fprintf(stderr, "Could not add subnet2 to subnets list\n");
+		exit(EXIT_FAILURE);
+	}
+
+	size = loc_network_list_size(subnets);
+	if (size != 2) {
+		fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
+		exit(EXIT_FAILURE);
+	}
+
+	// Exclude subnet1 from network1
+	struct loc_network_list* excluded = loc_network_exclude(network1, subnet1);
+	if (!excluded) {
+		fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n");
+		exit(EXIT_FAILURE);
+	}
+
+	loc_network_list_dump(excluded);
+
+	// Exclude all subnets from network1
+	excluded = loc_network_exclude_list(network1, subnets);
+	if (!excluded) {
+		fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n");
+		exit(EXIT_FAILURE);
+	}
+
+	loc_network_list_dump(excluded);
+
+	if (excluded)
+		loc_network_list_unref(excluded);
+
+	loc_network_list_unref(subnets);
+	loc_network_unref(network1);
+	loc_network_unref(subnet1);
+	loc_network_unref(subnet2);
+	loc_unref(ctx);
+
+	return EXIT_SUCCESS;
+}
-- 
2.20.1

From 1209ff0cd4be6b2f52a5e82168db8a8ac68e628a Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 20 Nov 2020 16:24:40 +0000
Subject: [PATCH 079/111] network: Fix loc_network_is_subnet()

This function always returned false.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/src/network.c b/src/network.c
index 4c8787a..b440d76 100644
--- a/src/network.c
+++ b/src/network.c
@@ -491,12 +491,12 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
 LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
 	// If the start address of the other network is smaller than this network,
 	// it cannot be a subnet.
-	if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
+	if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
 		return 0;
 
 	// If the end address of the other network is greater than this network,
 	// it cannot be a subnet.
-	if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
+	if (in6_addr_cmp(&self->last_address, &other->last_address) < 0)
 		return 0;
 
 	return 1;
@@ -504,17 +504,7 @@ LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_networ
 
 // XXX DEPRECATED - I find this too difficult to use
 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
-	// If the start address of the other network is smaller than this network,
-	// it cannot be a subnet.
-	if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
-		return 0;
-
-	// If the end address of the other network is greater than this network,
-	// it cannot be a subnet.
-	if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
-		return 0;
-
-	return 1;
+	return loc_network_is_subnet(other, self);
 }
 
 LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
-- 
2.20.1

From 92349c14876f8e4711739c82d1a389a53bcb27a5 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 20 Nov 2020 16:25:27 +0000
Subject: [PATCH 080/111] network: Remove debugging output

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/network.c b/src/network.c
index b440d76..3271272 100644
--- a/src/network.c
+++ b/src/network.c
@@ -670,10 +670,6 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude(
 		loc_network_unref(subnet2);
 	}
 
-#ifdef ENABLE_DEBUG
-	loc_network_list_dump(list);
-#endif
-
 	// Return the result
 	return list;
 
-- 
2.20.1

From ed0f53df0f3ebb915faf25138cc09df7555415a3 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 20 Nov 2020 16:25:56 +0000
Subject: [PATCH 081/111] database: Flatten out code due to compiler errors

It looks like GCC could not follow the code as it was
written. Therefore I clean up more often now instead
of having a GOTO block where I do that in.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 59 ++++++++++++++++++++++++++++++++++----------------
 1 file changed, 40 insertions(+), 19 deletions(-)

diff --git a/src/database.c b/src/database.c
index ef4f505..c21b957 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1305,8 +1305,12 @@ static int __loc_database_enumerator_next_network_flattened(
 	while (1) {
 		// Fetch the next network in line
 		r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
-		if (r)
-			goto END;
+		if (r) {
+			loc_network_unref(subnet);
+			loc_network_list_unref(subnets);
+
+			return r;
+		}
 
 		// End if we did not receive another subnet
 		if (!subnet)
@@ -1315,8 +1319,12 @@ static int __loc_database_enumerator_next_network_flattened(
 		// Collect all subnets in a list
 		if (loc_network_is_subnet(*network, subnet)) {
 			r = loc_network_list_push(subnets, subnet);
-			if (r)
-				goto END;
+			if (r) {
+				loc_network_unref(subnet);
+				loc_network_list_unref(subnets);
+
+				return r;
+			}
 
 			loc_network_unref(subnet);
 			continue;
@@ -1324,8 +1332,12 @@ static int __loc_database_enumerator_next_network_flattened(
 
 		// If this is not a subnet, we push it back onto the stack and break
 		r = loc_network_list_push(enumerator->stack, subnet);
-		if (r)
-			goto END;
+		if (r) {
+			loc_network_unref(subnet);
+			loc_network_list_unref(subnets);
+
+			return r;
+		}
 
 		loc_network_unref(subnet);
 		break;
@@ -1343,29 +1355,38 @@ static int __loc_database_enumerator_next_network_flattened(
 	// If the network has any subnets, we will break it into smaller parts
 	// without the subnets.
 	struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
-	if (!excluded || loc_network_list_empty(excluded)) {
-		r = 1;
-		goto END;
+	if (!excluded) {
+		loc_network_list_unref(subnets);
+		return -1;
+	}
+
+	// Merge excluded list with subnets
+	r = loc_network_list_merge(subnets, excluded);
+	if (r) {
+		loc_network_list_unref(subnets);
+		loc_network_list_unref(excluded);
+
+		return r;
 	}
 
+	// We no longer need the excluded list
+	loc_network_list_unref(excluded);
+
+	// Sort all subnets
+	loc_network_list_sort(subnets);
+
 	// Replace network with the first one
 	loc_network_unref(*network);
 
-	*network = loc_network_list_pop_first(excluded);
+	*network = loc_network_list_pop_first(subnets);
 
 	// Push the rest onto the stack
-	loc_network_list_reverse(excluded);
-	loc_network_list_merge(enumerator->stack, excluded);
-
-	loc_network_list_unref(excluded);
-
-END:
-	if (subnet)
-		loc_network_unref(subnet);
+	loc_network_list_reverse(subnets);
+	loc_network_list_merge(enumerator->stack, subnets);
 
 	loc_network_list_unref(subnets);
 
-	return r;
+	return 0;
 }
 
 LOC_EXPORT int loc_database_enumerator_next_network(
-- 
2.20.1

From a1024390795d60fab9f697fa230c9901ebd7d221 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 20 Nov 2020 16:42:55 +0000
Subject: [PATCH 082/111] network-list: Implement merging in reverse in one
 step

This will save us some time because we do not need to
change the list in place first and then merge it.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c         |  3 +--
 src/libloc.sym         |  1 +
 src/loc/network-list.h |  1 +
 src/network-list.c     | 13 +++++++++++++
 4 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/database.c b/src/database.c
index c21b957..ba4f98a 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1381,8 +1381,7 @@ static int __loc_database_enumerator_next_network_flattened(
 	*network = loc_network_list_pop_first(subnets);
 
 	// Push the rest onto the stack
-	loc_network_list_reverse(subnets);
-	loc_network_list_merge(enumerator->stack, subnets);
+	loc_network_list_merge_reverse(enumerator->stack, subnets);
 
 	loc_network_list_unref(subnets);
 
diff --git a/src/libloc.sym b/src/libloc.sym
index 406dd15..56cada8 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -140,6 +140,7 @@ global:
 	loc_network_list_empty;
 	loc_network_list_get;
 	loc_network_list_merge;
+	loc_network_list_merge_reverse;
 	loc_network_list_new;
 	loc_network_list_pop;
 	loc_network_list_pop_first;
diff --git a/src/loc/network-list.h b/src/loc/network-list.h
index af3b28d..89776a6 100644
--- a/src/loc/network-list.h
+++ b/src/loc/network-list.h
@@ -33,5 +33,6 @@ int loc_network_list_contains(struct loc_network_list* list, struct loc_network*
 void loc_network_list_sort(struct loc_network_list* list);
 void loc_network_list_reverse(struct loc_network_list* list);
 int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
+int loc_network_list_merge_reverse(struct loc_network_list* self, struct loc_network_list* other);
 
 #endif
diff --git a/src/network-list.c b/src/network-list.c
index 9cb4547..b455caf 100644
--- a/src/network-list.c
+++ b/src/network-list.c
@@ -242,3 +242,16 @@ LOC_EXPORT int loc_network_list_merge(
 
 	return 0;
 }
+
+LOC_EXPORT int loc_network_list_merge_reverse(
+		struct loc_network_list* self, struct loc_network_list* other) {
+	int r;
+
+	for (int i = other->size - 1; i >= 0; i--) {
+		r = loc_network_list_push(self, other->elements[i]);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
-- 
2.20.1

From a5967330d530504db401540d4bcd5474fe00e421 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 20 Nov 2020 18:36:48 +0000
Subject: [PATCH 083/111] network: Optimise _subnet function

This function used to create a network list which always
had exactly two elements. Since splitting a network in half
always returns two parts, we can simply return them as a
pointer.

This improves returning the network tree by about 17%.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/loc/network.h  |   2 +-
 src/network.c      | 158 ++++++++++++++++++---------------------------
 src/test-network.c |  51 ++++++++-------
 3 files changed, 91 insertions(+), 120 deletions(-)

diff --git a/src/loc/network.h b/src/loc/network.h
index 4b7410c..d67ec54 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -62,7 +62,7 @@ int loc_network_gt(struct loc_network* self, struct loc_network* other);
 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
-struct loc_network_list* loc_network_subnets(struct loc_network* network);
+int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
 struct loc_network_list* loc_network_exclude(
 		struct loc_network* self, struct loc_network* other);
 struct loc_network_list* loc_network_exclude_list(
diff --git a/src/network.c b/src/network.c
index 3271272..ff9c1eb 100644
--- a/src/network.c
+++ b/src/network.c
@@ -507,66 +507,91 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net
 	return loc_network_is_subnet(other, self);
 }
 
-LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
-	struct loc_network_list* list;
+LOC_EXPORT int loc_network_subnets(struct loc_network* network,
+		struct loc_network** subnet1, struct loc_network** subnet2) {
+	int r;
+	*subnet1 = NULL;
+	*subnet2 = NULL;
 
 	// New prefix length
 	unsigned int prefix = network->prefix + 1;
 
 	// Check if the new prefix is valid
 	if (valid_prefix(&network->first_address, prefix))
-		return NULL;
-
-	// Create a new list with the result
-	int r = loc_network_list_new(network->ctx, &list);
-	if (r) {
-		ERROR(network->ctx, "Could not create network list: %d\n", r);
-		return NULL;
-	}
-
-	struct loc_network* subnet1 = NULL;
-	struct loc_network* subnet2 = NULL;
+		return -1;
 
 	// Create the first half of the network
-	r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix);
+	r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix);
 	if (r)
-		goto ERROR;
+		return r;
 
 	// The next subnet starts after the first one
-	struct in6_addr first_address = address_increment(&subnet1->last_address);
+	struct in6_addr first_address = address_increment(&(*subnet1)->last_address);
 
 	// Create the second half of the network
-	r = loc_network_new(network->ctx, &subnet2, &first_address, prefix);
-	if (r)
-		goto ERROR;
-
-	// Push the both onto the stack (in reverse order)
-	r = loc_network_list_push(list, subnet2);
+	r = loc_network_new(network->ctx, subnet2, &first_address, prefix);
 	if (r)
-		goto ERROR;
-
-	r = loc_network_list_push(list, subnet1);
-	if (r)
-		goto ERROR;
+		return r;
 
 	// Copy country code
 	const char* country_code = loc_network_get_country_code(network);
 	if (country_code) {
-		loc_network_set_country_code(subnet1, country_code);
-		loc_network_set_country_code(subnet2, country_code);
+		loc_network_set_country_code(*subnet1, country_code);
+		loc_network_set_country_code(*subnet2, country_code);
 	}
 
 	// Copy ASN
 	uint32_t asn = loc_network_get_asn(network);
 	if (asn) {
-		loc_network_set_asn(subnet1, asn);
-		loc_network_set_asn(subnet2, asn);
+		loc_network_set_asn(*subnet1, asn);
+		loc_network_set_asn(*subnet2, asn);
 	}
 
-	loc_network_unref(subnet1);
-	loc_network_unref(subnet2);
+	return 0;
+}
 
-	return list;
+static int __loc_network_exclude(struct loc_network* network,
+		struct loc_network* other, struct loc_network_list* list) {
+	struct loc_network* subnet1 = NULL;
+	struct loc_network* subnet2 = NULL;
+
+	int r = loc_network_subnets(network, &subnet1, &subnet2);
+	if (r)
+		goto ERROR;
+
+	if (loc_network_eq(other, subnet1)) {
+		r = loc_network_list_push(list, subnet2);
+		if (r)
+			goto ERROR;
+
+	} else if (loc_network_eq(other, subnet2)) {
+		r = loc_network_list_push(list, subnet1);
+		if (r)
+			goto ERROR;
+
+	} else  if (loc_network_is_subnet_of(other, subnet1)) {
+		r = loc_network_list_push(list, subnet2);
+		if (r)
+			goto ERROR;
+
+		r = __loc_network_exclude(subnet1, other, list);
+		if (r)
+			goto ERROR;
+
+	} else if (loc_network_is_subnet_of(other, subnet2)) {
+		r = loc_network_list_push(list, subnet1);
+		if (r)
+			goto ERROR;
+
+		r = __loc_network_exclude(subnet2, other, list);
+		if (r)
+			goto ERROR;
+
+	} else {
+		ERROR(network->ctx, "We should never get here\n");
+		r = 1;
+		goto ERROR;
+	}
 
 ERROR:
 	if (subnet1)
@@ -575,10 +600,7 @@ ERROR:
 	if (subnet2)
 		loc_network_unref(subnet2);
 
-	if (list)
-		loc_network_list_unref(list);
-
-	return NULL;
+	return r;
 }
 
 LOC_EXPORT struct loc_network_list* loc_network_exclude(
@@ -623,67 +645,15 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude(
 		return NULL;
 	}
 
-	struct loc_network_list* subnets = loc_network_subnets(self);
-
-	struct loc_network* subnet1 = NULL;
-	struct loc_network* subnet2 = NULL;
-
-	while (subnets) {
-		// Fetch both subnets
-		subnet1 = loc_network_list_get(subnets, 0);
-		subnet2 = loc_network_list_get(subnets, 1);
-
-		// Free list
-		loc_network_list_unref(subnets);
-		subnets = NULL;
-
-		if (loc_network_eq(other, subnet1)) {
-			r = loc_network_list_push(list, subnet2);
-			if (r)
-				goto ERROR;
-
-		} else if (loc_network_eq(other, subnet2)) {
-			r = loc_network_list_push(list, subnet1);
-			if (r)
-				goto ERROR;
-
-		} else  if (loc_network_is_subnet_of(other, subnet1)) {
-			r = loc_network_list_push(list, subnet2);
-			if (r)
-				goto ERROR;
-
-			subnets = loc_network_subnets(subnet1);
-
-		} else if (loc_network_is_subnet_of(other, subnet2)) {
-			r = loc_network_list_push(list, subnet1);
-			if (r)
-				goto ERROR;
-
-			subnets = loc_network_subnets(subnet2);
-
-		} else {
-			ERROR(self->ctx, "We should never get here\n");
-			goto ERROR;
-		}
+	r = __loc_network_exclude(self, other, list);
+	if (r) {
+		loc_network_list_unref(list);
 
-		loc_network_unref(subnet1);
-		loc_network_unref(subnet2);
+		return NULL;
 	}
 
 	// Return the result
 	return list;
-
-ERROR:
-	if (subnet1)
-		loc_network_unref(subnet1);
-
-	if (subnet2)
-		loc_network_unref(subnet2);
-
-	if (list)
-		loc_network_list_unref(list);
-
-	return NULL;
 }
 
 LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
diff --git a/src/test-network.c b/src/test-network.c
index 7c90224..79c2967 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -138,47 +138,48 @@ int main(int argc, char** argv) {
 	}
 
 	// Check subnet function
-	err = loc_network_is_subnet_of(network1, network2);
-	if (err != 0) {
+	err = loc_network_is_subnet(network1, network2);
+	if (!err) {
 		fprintf(stderr, "Subnet check 1 failed: %d\n", err);
 		exit(EXIT_FAILURE);
 	}
 
-	err = loc_network_is_subnet_of(network2, network1);
-	if (err != 1) {
+	err = loc_network_is_subnet(network2, network1);
+	if (err) {
 		fprintf(stderr, "Subnet check 2 failed: %d\n", err);
 		exit(EXIT_FAILURE);
 	}
 
-	// Make list of subnets
-	struct loc_network_list* subnets = loc_network_subnets(network1);
-	if (!subnets) {
-		fprintf(stderr, "Could not find subnets of network\n");
+	// Make subnets
+	struct loc_network* subnet1 = NULL;
+	struct loc_network* subnet2 = NULL;
+
+	err  = loc_network_subnets(network1, &subnet1, &subnet2);
+	if (err || !subnet1 || !subnet2) {
+		fprintf(stderr, "Could not find subnets of network: %d\n", err);
 		exit(EXIT_FAILURE);
 	}
 
-	loc_network_list_dump(subnets);
-
-	while (!loc_network_list_empty(subnets)) {
-		struct loc_network* subnet = loc_network_list_pop(subnets);
-		if (!subnet) {
-			fprintf(stderr, "Received an empty subnet\n");
-			exit(EXIT_FAILURE);
-		}
+	char* s = loc_network_str(subnet1);
+	printf("Received subnet1 = %s\n", s);
+	free(s);
 
-		char* s = loc_network_str(subnet);
-		printf("Received subnet %s\n", s);
-		free(s);
+	s = loc_network_str(subnet2);
+	printf("Received subnet2 = %s\n", s);
+	free(s);
 
-		if (!loc_network_is_subnet_of(subnet, network1)) {
-			fprintf(stderr, "Not a subnet\n");
-			exit(EXIT_FAILURE);
-		}
+	if (!loc_network_is_subnet(network1, subnet1)) {
+		fprintf(stderr, "Subnet1 is not a subnet\n");
+		exit(EXIT_FAILURE);
+	}
 
-		loc_network_unref(subnet);
+	if (!loc_network_is_subnet(network1, subnet2)) {
+		fprintf(stderr, "Subnet2 is not a subnet\n");
+		exit(EXIT_FAILURE);
 	}
 
-	loc_network_list_unref(subnets);
+	loc_network_unref(subnet1);
+	loc_network_unref(subnet2);
 
 	struct loc_network_list* excluded = loc_network_exclude(network1, network2);
 	if (!excluded) {
-- 
2.20.1

From 7fe6a21845edf6692d239f228117bc95620d0419 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 20 Nov 2020 18:38:47 +0000
Subject: [PATCH 084/111] network: Add function to return the prefix

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym    |  1 +
 src/loc/network.h |  1 +
 src/network.c     | 12 ++++++++++++
 3 files changed, 14 insertions(+)

diff --git a/src/libloc.sym b/src/libloc.sym
index 56cada8..9f41c89 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -125,6 +125,7 @@ global:
 	loc_network_new;
 	loc_network_new_from_string;
 	loc_network_overlaps;
+	loc_network_prefix;
 	loc_network_ref;
 	loc_network_set_asn;
 	loc_network_set_country_code;
diff --git a/src/loc/network.h b/src/loc/network.h
index d67ec54..7b2ae4c 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -38,6 +38,7 @@ struct loc_network* loc_network_ref(struct loc_network* network);
 struct loc_network* loc_network_unref(struct loc_network* network);
 char* loc_network_str(struct loc_network* network);
 int loc_network_address_family(struct loc_network* network);
+unsigned int loc_network_prefix(struct loc_network* network);
 
 const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
 char* loc_network_format_first_address(struct loc_network* network);
diff --git a/src/network.c b/src/network.c
index ff9c1eb..4720503 100644
--- a/src/network.c
+++ b/src/network.c
@@ -310,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) {
 	return network->family;
 }
 
+LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) {
+	switch (network->family) {
+		case AF_INET6:
+			return network->prefix;
+
+		case AF_INET:
+			return network->prefix - 96;
+	}
+
+	return 0;
+}
+
 static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) {
 	const size_t length = INET6_ADDRSTRLEN;
 
-- 
2.20.1

From cf8d3c6454843943e3bc81eb85522779d1c11f9b Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 20 Nov 2020 18:39:13 +0000
Subject: [PATCH 085/111] network: Speed up subnet check

There is no point in checking different address families
with each other and we do not need to compare addresses
when the prefix of the subnet does not fit into the
network to check.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/network.c b/src/network.c
index 4720503..e52b58c 100644
--- a/src/network.c
+++ b/src/network.c
@@ -501,6 +501,14 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
 }
 
 LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
+	// Check family
+	if (self->family != other->family)
+		return 0;
+
+	// The prefix must be smaller (this avoids the more complex comparisons later)
+	if (self->prefix > other->prefix)
+		return 0;
+
 	// If the start address of the other network is smaller than this network,
 	// it cannot be a subnet.
 	if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
-- 
2.20.1

From d6a5092f969bb3bd50d130d4ba64b4e4be2e61f6 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 20 Nov 2020 18:44:42 +0000
Subject: [PATCH 086/111] network: Remove deprecated loc_network_is_subnet_of
 function

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym       |  1 -
 src/loc/network.h    |  1 -
 src/network.c        | 13 ++++---------
 src/python/network.c |  2 +-
 4 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/libloc.sym b/src/libloc.sym
index 9f41c89..b2d8a31 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -118,7 +118,6 @@ global:
 	loc_network_gt;
 	loc_network_has_flag;
 	loc_network_is_subnet;
-	loc_network_is_subnet_of;
 	loc_network_match_asn;
 	loc_network_match_country_code;
 	loc_network_match_flag;
diff --git a/src/loc/network.h b/src/loc/network.h
index 7b2ae4c..b31c8a2 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -62,7 +62,6 @@ int loc_network_eq(struct loc_network* self, struct loc_network* other);
 int loc_network_gt(struct loc_network* self, struct loc_network* other);
 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
-int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
 int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
 struct loc_network_list* loc_network_exclude(
 		struct loc_network* self, struct loc_network* other);
diff --git a/src/network.c b/src/network.c
index e52b58c..72b77e6 100644
--- a/src/network.c
+++ b/src/network.c
@@ -522,11 +522,6 @@ LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_networ
 	return 1;
 }
 
-// XXX DEPRECATED - I find this too difficult to use
-LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
-	return loc_network_is_subnet(other, self);
-}
-
 LOC_EXPORT int loc_network_subnets(struct loc_network* network,
 		struct loc_network** subnet1, struct loc_network** subnet2) {
 	int r;
@@ -589,7 +584,7 @@ static int __loc_network_exclude(struct loc_network* network,
 		if (r)
 			goto ERROR;
 
-	} else  if (loc_network_is_subnet_of(other, subnet1)) {
+	} else  if (loc_network_is_subnet(subnet1, other)) {
 		r = loc_network_list_push(list, subnet2);
 		if (r)
 			goto ERROR;
@@ -598,7 +593,7 @@ static int __loc_network_exclude(struct loc_network* network,
 		if (r)
 			goto ERROR;
 
-	} else if (loc_network_is_subnet_of(other, subnet2)) {
+	} else if (loc_network_is_subnet(subnet2, other)) {
 		r = loc_network_list_push(list, subnet1);
 		if (r)
 			goto ERROR;
@@ -645,7 +640,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude(
 	}
 
 	// Other must be a subnet of self
-	if (!loc_network_is_subnet_of(other, self)) {
+	if (!loc_network_is_subnet(self, other)) {
 		DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
 
 		return NULL;
@@ -726,7 +721,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
 			}
 
 			// Drop this subnet if is a subnet of another subnet
-			if (loc_network_is_subnet_of(subnet, subnet_to_check)) {
+			if (loc_network_is_subnet(subnet_to_check, subnet)) {
 				passed = 0;
 				loc_network_unref(subnet);
 				break;
diff --git a/src/python/network.c b/src/python/network.c
index 742b472..b6e92fb 100644
--- a/src/python/network.c
+++ b/src/python/network.c
@@ -194,7 +194,7 @@ static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
 	if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
 		return NULL;
 
-	if (loc_network_is_subnet_of(self->network, other->network))
+	if (loc_network_is_subnet(other->network, self->network))
 		Py_RETURN_TRUE;
 
 	Py_RETURN_FALSE;
-- 
2.20.1

From af4689bf5c56ad79e9e90396b41be460e49ef384 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 24 Nov 2020 14:55:08 +0000
Subject: [PATCH 087/111] network-list: Make this a sorted list

The list is now organised so that it is always ordered.

This allows us to find if a network is on the list a lot
quicker using binary search as well as inserting a new
network at the right place.

This will benefit loc_network_exclude with very large networks.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym     |  1 +
 src/loc/network.h  |  1 +
 src/network-list.c | 84 +++++++++++++++++++++++++++++++++++++++++-----
 src/network.c      | 22 ++++++++++++
 4 files changed, 99 insertions(+), 9 deletions(-)

diff --git a/src/libloc.sym b/src/libloc.sym
index b2d8a31..28cc8e8 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -106,6 +106,7 @@ global:
 
 	# Network
 	loc_network_address_family;
+	loc_network_cmp;
 	loc_network_eq;
 	loc_network_exclude;
 	loc_network_exclude_list;
diff --git a/src/loc/network.h b/src/loc/network.h
index b31c8a2..8ab1562 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -58,6 +58,7 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
 
+int loc_network_cmp(struct loc_network* self, struct loc_network* other);
 int loc_network_eq(struct loc_network* self, struct loc_network* other);
 int loc_network_gt(struct loc_network* self, struct loc_network* other);
 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
diff --git a/src/network-list.c b/src/network-list.c
index b455caf..6e9cd37 100644
--- a/src/network-list.c
+++ b/src/network-list.c
@@ -16,6 +16,7 @@
 
 #include <errno.h>
 #include <stdlib.h>
+#include <time.h>
 
 #include <loc/libloc.h>
 #include <loc/network.h>
@@ -130,11 +131,71 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
 	return loc_network_ref(list->elements[index]);
 }
 
+//MOVE FUNCTION GOES HERE
+
+static off_t loc_network_list_find(struct loc_network_list* list,
+		struct loc_network* network, int* found) {
+	off_t lo = 0;
+	off_t hi = list->size - 1;
+
+	*found = 0;
+
+#ifdef ENABLE_DEBUG
+	// Save start time
+	clock_t start = clock();
+#endif
+
+	off_t i = 0;
+
+	while (lo <= hi) {
+		i = (lo + hi) / 2;
+
+		// Check if this is a match
+		int result = loc_network_cmp(network, list->elements[i]);
+
+		if (result == 0) {
+			*found = 1;
+
+#ifdef ENABLE_DEBUG
+			clock_t end = clock();
+
+			// Log how fast this has been
+			DEBUG(list->ctx, "Found network in %.4fms at %jd\n",
+				(double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
+#endif
+
+			return i;
+		}
+
+		if (result > 0)
+			lo = i + 1;
+		else
+			hi = i - 1;
+	}
+
+#ifdef ENABLE_DEBUG
+	clock_t end = clock();
+
+	// Log how fast this has been
+	DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n",
+		(double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
+#endif
+
+	return i;
+}
+
 LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
-	// Do not add networks that are already on the list
-	if (loc_network_list_contains(list, network))
+	int found = 0;
+
+	off_t index = loc_network_list_find(list, network, &found);
+
+	// The network has been found on the list. Nothing to do.
+	if (found)
 		return 0;
 
+	DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n",
+		list, network, (intmax_t)index);
+
 	// Check if we have space left
 	if (list->size >= list->elements_size) {
 		int r = loc_network_list_grow(list, 64);
@@ -142,9 +203,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
 			return r;
 	}
 
-	DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
+	// The list is now larger
+	list->size++;
+
+	// Move all elements out of the way
+	for (unsigned int i = list->size - 1; i > index; i--)
+		list->elements[i] = list->elements[i - 1];
 
-	list->elements[list->size++] = loc_network_ref(network);
+	// Add the new element at the right place
+	list->elements[index] = loc_network_ref(network);
 
 	return 0;
 }
@@ -183,12 +250,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
 }
 
 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
-	for (unsigned int i = 0; i < list->size; i++) {
-		if (loc_network_eq(list->elements[i], network))
-			return 1;
-	}
+	int found = 0;
 
-	return 0;
+	loc_network_list_find(list, network, &found);
+
+	return found;
 }
 
 static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
diff --git a/src/network.c b/src/network.c
index 72b77e6..38d557a 100644
--- a/src/network.c
+++ b/src/network.c
@@ -441,6 +441,28 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
 	return loc_network_has_flag(network, flag);
 }
 
+LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
+	// Compare family
+	if (self->family > other->family)
+		return 1;
+	else if (self->family < other->family)
+		return -1;
+
+	// Compare address
+	int r = in6_addr_cmp(&self->first_address, &other->first_address);
+	if (r)
+		return r;
+
+	// Compare prefix
+	if (self->prefix > other->prefix)
+		return 1;
+	else if (self->prefix < other->prefix)
+		return -1;
+
+	// Both networks are equal
+	return 0;
+}
+
 LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
 	// Family must be the same
 	if (self->family != other->family)
-- 
2.20.1

From 39cbbc63aee362d82f69a9b4722b59153ce799a0 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 24 Nov 2020 15:04:03 +0000
Subject: [PATCH 088/111] network-list: Drop sorting functions

Since the list is always sorted, there is no point in
sorting it again...

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c         |  6 +----
 src/libloc.sym         |  3 ---
 src/loc/network-list.h |  3 ---
 src/network-list.c     | 52 ------------------------------------------
 src/test-network.c     |  9 --------
 5 files changed, 1 insertion(+), 72 deletions(-)

diff --git a/src/database.c b/src/database.c
index ba4f98a..5546091 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1372,17 +1372,13 @@ static int __loc_database_enumerator_next_network_flattened(
 	// We no longer need the excluded list
 	loc_network_list_unref(excluded);
 
-	// Sort all subnets
-	loc_network_list_sort(subnets);
-
 	// Replace network with the first one
 	loc_network_unref(*network);
 
 	*network = loc_network_list_pop_first(subnets);
 
 	// Push the rest onto the stack
-	loc_network_list_merge_reverse(enumerator->stack, subnets);
-
+	loc_network_list_merge(enumerator->stack, subnets);
 	loc_network_list_unref(subnets);
 
 	return 0;
diff --git a/src/libloc.sym b/src/libloc.sym
index 28cc8e8..4b0ce45 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -141,15 +141,12 @@ global:
 	loc_network_list_empty;
 	loc_network_list_get;
 	loc_network_list_merge;
-	loc_network_list_merge_reverse;
 	loc_network_list_new;
 	loc_network_list_pop;
 	loc_network_list_pop_first;
 	loc_network_list_push;
 	loc_network_list_ref;
-	loc_network_list_reverse;
 	loc_network_list_size;
-	loc_network_list_sort;
 	loc_network_list_unref;
 
 	# Writer
diff --git a/src/loc/network-list.h b/src/loc/network-list.h
index 89776a6..21c7402 100644
--- a/src/loc/network-list.h
+++ b/src/loc/network-list.h
@@ -30,9 +30,6 @@ int loc_network_list_push(struct loc_network_list* list, struct loc_network* net
 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
 struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
 int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
-void loc_network_list_sort(struct loc_network_list* list);
-void loc_network_list_reverse(struct loc_network_list* list);
 int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
-int loc_network_list_merge_reverse(struct loc_network_list* self, struct loc_network_list* other);
 
 #endif
diff --git a/src/network-list.c b/src/network-list.c
index 6e9cd37..7e8b5f3 100644
--- a/src/network-list.c
+++ b/src/network-list.c
@@ -257,45 +257,6 @@ LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct l
 	return found;
 }
 
-static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
-	// Do nothing for invalid indices
-	if (i1 >= list->size || i2 >= list->size)
-		return;
-
-	struct loc_network* network1 = list->elements[i1];
-	struct loc_network* network2 = list->elements[i2];
-
-	list->elements[i1] = network2;
-	list->elements[i2] = network1;
-}
-
-LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
-	unsigned int i = 0;
-	unsigned int j = list->size - 1;
-
-	while (i < j) {
-		loc_network_list_swap(list, i++, j--);
-	}
-}
-
-LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
-	unsigned int n = list->size;
-	int swapped;
-
-	do {
-		swapped = 0;
-
-		for (unsigned int i = 1; i < n; i++) {
-			if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) {
-				loc_network_list_swap(list, i-1, i);
-				swapped = 1;
-			}
-		}
-
-		n--;
-	} while (swapped);
-}
-
 LOC_EXPORT int loc_network_list_merge(
 		struct loc_network_list* self, struct loc_network_list* other) {
 	int r;
@@ -308,16 +269,3 @@ LOC_EXPORT int loc_network_list_merge(
 
 	return 0;
 }
-
-LOC_EXPORT int loc_network_list_merge_reverse(
-		struct loc_network_list* self, struct loc_network_list* other) {
-	int r;
-
-	for (int i = other->size - 1; i >= 0; i--) {
-		r = loc_network_list_push(self, other->elements[i]);
-		if (r)
-			return r;
-	}
-
-	return 0;
-}
diff --git a/src/test-network.c b/src/test-network.c
index 79c2967..8a6763c 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -188,15 +188,6 @@ int main(int argc, char** argv) {
 	}
 
 	loc_network_list_dump(excluded);
-
-	// Reverse it
-	loc_network_list_reverse(excluded);
-	loc_network_list_dump(excluded);
-
-	// Sort them and dump them again
-	loc_network_list_sort(excluded);
-	loc_network_list_dump(excluded);
-
 	loc_network_list_unref(excluded);
 
 	// Create a database
-- 
2.20.1

From 5dacb45afceac2d05ea597755c1ca5a1b62cc0fd Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 24 Nov 2020 15:15:59 +0000
Subject: [PATCH 089/111] database: Avoid merging the same data twice

When finish splitting networks into many parts, we have
a list of subnets with the excluded subnets and merge them
together first and put them on the stack again.

This is slower than pushing it all onto the stack first
and then popping the first element.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/database.c b/src/database.c
index 5546091..f1f6ae0 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1360,8 +1360,8 @@ static int __loc_database_enumerator_next_network_flattened(
 		return -1;
 	}
 
-	// Merge excluded list with subnets
-	r = loc_network_list_merge(subnets, excluded);
+	// Merge subnets onto the stack
+	r = loc_network_list_merge(enumerator->stack, subnets);
 	if (r) {
 		loc_network_list_unref(subnets);
 		loc_network_list_unref(excluded);
@@ -1369,17 +1369,21 @@ static int __loc_database_enumerator_next_network_flattened(
 		return r;
 	}
 
-	// We no longer need the excluded list
-	loc_network_list_unref(excluded);
-
-	// Replace network with the first one
-	loc_network_unref(*network);
+	// Push excluded list onto the stack
+	r = loc_network_list_merge(enumerator->stack, excluded);
+	if (r) {
+		loc_network_list_unref(subnets);
+		loc_network_list_unref(excluded);
 
-	*network = loc_network_list_pop_first(subnets);
+		return r;
+	}
 
-	// Push the rest onto the stack
-	loc_network_list_merge(enumerator->stack, subnets);
 	loc_network_list_unref(subnets);
+	loc_network_list_unref(excluded);
+
+	// Replace network with the first one from the stack
+	loc_network_unref(*network);
+	*network = loc_network_list_pop_first(enumerator->stack);
 
 	return 0;
 }
-- 
2.20.1

From 04cbd2bfa892fc7374ad506ec7ba81727c5a4b96 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 24 Nov 2020 15:22:02 +0000
Subject: [PATCH 090/111] database: Read the first element from the list

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/database.c b/src/database.c
index f1f6ae0..914ed3e 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1191,7 +1191,7 @@ static int __loc_database_enumerator_next_network(
 		struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
 	// Return top element from the stack
 	while (1) {
-		*network = loc_network_list_pop(enumerator->stack);
+		*network = loc_network_list_pop_first(enumerator->stack);
 
 		// Stack is empty
 		if (!*network)
-- 
2.20.1

From c650008e2dd9af5fd1eadba6354aa0b615047f84 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 24 Nov 2020 15:41:53 +0000
Subject: [PATCH 091/111] network: Add excluded networks to to_check list

This is now being done immediately instead of creating a new
list which is being merged later.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 55 ++++++++++++++++++++++++++++++---------------------
 1 file changed, 32 insertions(+), 23 deletions(-)

diff --git a/src/network.c b/src/network.c
index 38d557a..7ab22f8 100644
--- a/src/network.c
+++ b/src/network.c
@@ -640,49 +640,55 @@ ERROR:
 	return r;
 }
 
-LOC_EXPORT struct loc_network_list* loc_network_exclude(
-		struct loc_network* self, struct loc_network* other) {
-	struct loc_network_list* list;
-
-#ifdef ENABLE_DEBUG
-	char* n1 = loc_network_str(self);
-	char* n2 = loc_network_str(other);
-
-	DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
-
-	free(n1);
-	free(n2);
-#endif
-
+static int __loc_network_exclude_to_list(struct loc_network* self,
+		struct loc_network* other, struct loc_network_list* list) {
 	// Family must match
 	if (self->family != other->family) {
 		DEBUG(self->ctx, "Family mismatch\n");
 
-		return NULL;
+		return 1;
 	}
 
 	// Other must be a subnet of self
 	if (!loc_network_is_subnet(self, other)) {
 		DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
 
-		return NULL;
+		return 1;
 	}
 
 	// We cannot perform this operation if both networks equal
 	if (loc_network_eq(self, other)) {
 		DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
 
-		return NULL;
+		return 1;
 	}
 
+	return __loc_network_exclude(self, other, list);
+}
+
+LOC_EXPORT struct loc_network_list* loc_network_exclude(
+		struct loc_network* self, struct loc_network* other) {
+	struct loc_network_list* list;
+
+#ifdef ENABLE_DEBUG
+	char* n1 = loc_network_str(self);
+	char* n2 = loc_network_str(other);
+
+	DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
+
+	free(n1);
+	free(n2);
+#endif
+
 	// Create a new list with the result
 	int r = loc_network_list_new(self->ctx, &list);
 	if (r) {
 		ERROR(self->ctx, "Could not create network list: %d\n", r);
+
 		return NULL;
 	}
 
-	r = __loc_network_exclude(self, other, list);
+	r = __loc_network_exclude_to_list(self, other, list);
 	if (r) {
 		loc_network_list_unref(list);
 
@@ -709,11 +715,14 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
 		subnet = loc_network_list_get(list, i);
 
 		// Find all excluded networks
-		struct loc_network_list* excluded = loc_network_exclude(network, subnet);
-		if (excluded) {
-			// Add them all to the "to check" list
-			loc_network_list_merge(to_check, excluded);
-			loc_network_list_unref(excluded);
+		if (!loc_network_list_contains(to_check, subnet)) {
+			r = __loc_network_exclude_to_list(network, subnet, to_check);
+			if (r) {
+				loc_network_list_unref(to_check);
+				loc_network_unref(subnet);
+
+				return NULL;
+			}
 		}
 
 		// Cleanup
-- 
2.20.1

From 7c6983ad52724d395446bdbd24d36b2ce22aecfd Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 24 Nov 2020 15:42:54 +0000
Subject: [PATCH 092/111] test: Add more networks to list to see the algorithm
 work

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/test-network-list.c | 32 +++++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/test-network-list.c b/src/test-network-list.c
index 3061d63..8253fc7 100644
--- a/src/test-network-list.c
+++ b/src/test-network-list.c
@@ -58,6 +58,20 @@ int main(int argc, char** argv) {
 		exit(EXIT_FAILURE);
 	}
 
+	struct loc_network* subnet3;
+	err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48");
+	if (err) {
+		fprintf(stderr, "Could not create the subnet3\n");
+		exit(EXIT_FAILURE);
+	}
+
+	struct loc_network* subnet4;
+	err = loc_network_new_from_string(ctx, &subnet4, "2001:db8:d::/48");
+	if (err) {
+		fprintf(stderr, "Could not create the subnet4\n");
+		exit(EXIT_FAILURE);
+	}
+
 	// Make a list with both subnets
 	struct loc_network_list* subnets;
 	err = loc_network_list_new(ctx, &subnets);
@@ -89,8 +103,24 @@ int main(int argc, char** argv) {
 		exit(EXIT_FAILURE);
 	}
 
+	// Add the fourth one next
+	err = loc_network_list_push(subnets, subnet4);
+	if (err) {
+		fprintf(stderr, "Could not add subnet4 to subnets list\n");
+		exit(EXIT_FAILURE);
+	}
+
+	// Add the third one
+	err = loc_network_list_push(subnets, subnet3);
+	if (err) {
+		fprintf(stderr, "Could not add subnet3 to subnets list\n");
+		exit(EXIT_FAILURE);
+	}
+
+	loc_network_list_dump(subnets);
+
 	size = loc_network_list_size(subnets);
-	if (size != 2) {
+	if (size != 4) {
 		fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
 		exit(EXIT_FAILURE);
 	}
-- 
2.20.1

From da101d55e66ebaeeed8b0b16829a2022a1af0678 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 24 Nov 2020 15:48:55 +0000
Subject: [PATCH 093/111] Drop loc_network_eq in favour of loc_network_cmp

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym     |  1 -
 src/loc/network.h  |  1 -
 src/network.c      | 24 ++++--------------------
 src/test-network.c |  8 ++++----
 4 files changed, 8 insertions(+), 26 deletions(-)

diff --git a/src/libloc.sym b/src/libloc.sym
index 4b0ce45..d8e8f14 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -107,7 +107,6 @@ global:
 	# Network
 	loc_network_address_family;
 	loc_network_cmp;
-	loc_network_eq;
 	loc_network_exclude;
 	loc_network_exclude_list;
 	loc_network_format_first_address;
diff --git a/src/loc/network.h b/src/loc/network.h
index 8ab1562..d5d0ccd 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -59,7 +59,6 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
 
 int loc_network_cmp(struct loc_network* self, struct loc_network* other);
-int loc_network_eq(struct loc_network* self, struct loc_network* other);
 int loc_network_gt(struct loc_network* self, struct loc_network* other);
 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
diff --git a/src/network.c b/src/network.c
index 7ab22f8..503bf3d 100644
--- a/src/network.c
+++ b/src/network.c
@@ -463,22 +463,6 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth
 	return 0;
 }
 
-LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
-	// Family must be the same
-	if (self->family != other->family)
-		return 0;
-
-	// The start address must be the same
-	if (in6_addr_cmp(&self->first_address, &other->first_address) != 0)
-		return 0;
-
-	// The prefix length must be the same
-	if (self->prefix != other->prefix)
-		return 0;
-
-	return 1;
-}
-
 LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
 	// Families must match
 	if (self->family != other->family)
@@ -596,12 +580,12 @@ static int __loc_network_exclude(struct loc_network* network,
 	if (r)
 		goto ERROR;
 
-	if (loc_network_eq(other, subnet1)) {
+	if (loc_network_cmp(other, subnet1) == 0) {
 		r = loc_network_list_push(list, subnet2);
 		if (r)
 			goto ERROR;
 
-	} else if (loc_network_eq(other, subnet2)) {
+	} else if (loc_network_cmp(other, subnet2) == 0) {
 		r = loc_network_list_push(list, subnet1);
 		if (r)
 			goto ERROR;
@@ -657,7 +641,7 @@ static int __loc_network_exclude_to_list(struct loc_network* self,
 	}
 
 	// We cannot perform this operation if both networks equal
-	if (loc_network_eq(self, other)) {
+	if (loc_network_cmp(self, other) == 0) {
 		DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
 
 		return 1;
@@ -745,7 +729,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
 			subnet = loc_network_list_get(list, i);
 
 			// Drop this subnet if is is already in list
-			if (loc_network_eq(subnet_to_check, subnet)) {
+			if (loc_network_cmp(subnet_to_check, subnet) == 0) {
 				passed = 0;
 				loc_network_unref(subnet);
 				break;
diff --git a/src/test-network.c b/src/test-network.c
index 8a6763c..f4cf97b 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -125,14 +125,14 @@ int main(int argc, char** argv) {
 #endif
 
 	// Check equals function
-	err = loc_network_eq(network1, network1);
-	if (!err) {
+	err = loc_network_cmp(network1, network1);
+	if (err) {
 		fprintf(stderr, "Network is not equal with itself\n");
 		exit(EXIT_FAILURE);
 	}
 
-	err = loc_network_eq(network1, network2);
-	if (err) {
+	err = loc_network_cmp(network1, network2);
+	if (!err) {
 		fprintf(stderr, "Networks equal unexpectedly\n");
 		exit(EXIT_FAILURE);
 	}
-- 
2.20.1

From 61a6f6e4bf4493236d796e94f3d721bcf5ba68aa Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 24 Nov 2020 15:50:39 +0000
Subject: [PATCH 094/111] Drop loc_network_gt which is now unused

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/libloc.sym    |  1 -
 src/loc/network.h |  1 -
 src/network.c     | 27 ---------------------------
 3 files changed, 29 deletions(-)

diff --git a/src/libloc.sym b/src/libloc.sym
index d8e8f14..cb5e8ef 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -115,7 +115,6 @@ global:
 	loc_network_get_country_code;
 	loc_network_get_first_address;
 	loc_network_get_last_address;
-	loc_network_gt;
 	loc_network_has_flag;
 	loc_network_is_subnet;
 	loc_network_match_asn;
diff --git a/src/loc/network.h b/src/loc/network.h
index d5d0ccd..af3dafd 100644
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -59,7 +59,6 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
 
 int loc_network_cmp(struct loc_network* self, struct loc_network* other);
-int loc_network_gt(struct loc_network* self, struct loc_network* other);
 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
 int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
diff --git a/src/network.c b/src/network.c
index 503bf3d..ac478d5 100644
--- a/src/network.c
+++ b/src/network.c
@@ -463,33 +463,6 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth
 	return 0;
 }
 
-LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
-	// Families must match
-	if (self->family != other->family)
-		return -1;
-
-	int r = in6_addr_cmp(&self->first_address, &other->first_address);
-
-	switch (r) {
-		// Smaller
-		case -1:
-			return 0;
-
-		// Larger
-		case 1:
-			return 1;
-
-		default:
-			break;
-	}
-
-	if (self->prefix > other->prefix)
-		return 1;
-
-	// Dunno
-	return 0;
-}
-
 LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
 	if (loc_network_match_address(self, &other->first_address) == 0)
 		return 1;
-- 
2.20.1

From fc692a58d9f4ca958f88cfa202250c572a0af6ea Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 24 Nov 2020 16:50:17 +0000
Subject: [PATCH 095/111] network: Adjust return codes of
 loc_network_match_address and add test

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/database.c     |  3 +--
 src/libloc.sym     |  1 +
 src/network.c      | 16 +++++++++-------
 src/test-network.c | 14 ++++++++++++++
 4 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/src/database.c b/src/database.c
index 914ed3e..1871b74 100644
--- a/src/database.c
+++ b/src/database.c
@@ -776,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru
 	}
 
 	// Check if the given IP address is inside the network
-	r = loc_network_match_address(*network, address);
-	if (r) {
+	if (!loc_network_match_address(*network, address)) {
 		DEBUG(db->ctx, "Searched address is not part of the network\n");
 
 		loc_network_unref(*network);
diff --git a/src/libloc.sym b/src/libloc.sym
index cb5e8ef..ee333f1 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -117,6 +117,7 @@ global:
 	loc_network_get_last_address;
 	loc_network_has_flag;
 	loc_network_is_subnet;
+	loc_network_match_address;
 	loc_network_match_asn;
 	loc_network_match_country_code;
 	loc_network_match_flag;
diff --git a/src/network.c b/src/network.c
index ac478d5..febab95 100644
--- a/src/network.c
+++ b/src/network.c
@@ -374,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
 LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) {
 	// Address must be larger than the start address
 	if (in6_addr_cmp(&network->first_address, address) > 0)
-		return 1;
+		return 0;
 
 	// Address must be smaller than the last address
 	if (in6_addr_cmp(&network->last_address, address) < 0)
-		return 1;
+		return 0;
 
 	// The address is inside this network
-	return 0;
+	return 1;
 }
 
 LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) {
@@ -464,16 +464,18 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth
 }
 
 LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
-	if (loc_network_match_address(self, &other->first_address) == 0)
+	// Either of the start addresses must be in the other subnet
+	if (loc_network_match_address(self, &other->first_address))
 		return 1;
 
-	if (loc_network_match_address(self, &other->last_address) == 0)
+	if (loc_network_match_address(other, &self->first_address))
 		return 1;
 
-	if (loc_network_match_address(other, &self->first_address) == 0)
+	// Or either of the end addresses is in the other subnet
+	if (loc_network_match_address(self, &other->last_address))
 		return 1;
 
-	if (loc_network_match_address(other, &self->last_address) == 0)
+	if (loc_network_match_address(other, &self->last_address))
 		return 1;
 
 	return 0;
diff --git a/src/test-network.c b/src/test-network.c
index f4cf97b..339743d 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -14,6 +14,7 @@
 	GNU General Public License for more details.
 */
 
+#include <arpa/inet.h>
 #include <errno.h>
 #include <stdio.h>
 #include <stddef.h>
@@ -46,6 +47,13 @@ int main(int argc, char** argv) {
 	}
 #endif
 
+	struct in6_addr address;
+	err = inet_pton(AF_INET6, "2001:db8::1", &address);
+	if (err != 1) {
+		fprintf(stderr, "Could not parse IP address\n");
+		exit(EXIT_FAILURE);
+	}
+
 	// Create a network
 	struct loc_network* network1;
 	err = loc_network_new_from_string(ctx, &network1, "2001:db8::1/32");
@@ -92,6 +100,12 @@ int main(int argc, char** argv) {
 		exit(EXIT_FAILURE);
 	}
 
+	err = loc_network_match_address(network1, &address);
+	if (!err) {
+		fprintf(stderr, "Network1 does not match address\n");
+		exit(EXIT_FAILURE);
+	}
+
 	struct loc_network* network2;
 	err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48");
 	if (err) {
-- 
2.20.1

From 06177d8c6004bf8b54322d92926152a7656f9c2a Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 24 Nov 2020 16:57:28 +0000
Subject: [PATCH 096/111] network-list: Use binary search to find if a network
 is a subnet

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/network.c b/src/network.c
index febab95..394bafc 100644
--- a/src/network.c
+++ b/src/network.c
@@ -697,19 +697,18 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
 	while (!loc_network_list_empty(to_check)) {
 		struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
 
+		// Check whether the subnet to check is part of the input list
+		if (loc_network_list_contains(list, subnet_to_check)) {
+			loc_network_unref(subnet_to_check);
+			continue;
+		}
+
 		// Marks whether this subnet passed all checks
 		int passed = 1;
 
 		for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
 			subnet = loc_network_list_get(list, i);
 
-			// Drop this subnet if is is already in list
-			if (loc_network_cmp(subnet_to_check, subnet) == 0) {
-				passed = 0;
-				loc_network_unref(subnet);
-				break;
-			}
-
 			// Drop this subnet if is a subnet of another subnet
 			if (loc_network_is_subnet(subnet_to_check, subnet)) {
 				passed = 0;
-- 
2.20.1

From 77e6d5379ea0de3264be5b8918d620d16e6bcbd3 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 24 Nov 2020 19:39:35 +0000
Subject: [PATCH 097/111] network-list: Check last element before doing binary
 search

This is helpful because very often we walk through a list in
order and are most interested in the last element.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network-list.c | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/network-list.c b/src/network-list.c
index 7e8b5f3..c86ad07 100644
--- a/src/network-list.c
+++ b/src/network-list.c
@@ -137,8 +137,26 @@ static off_t loc_network_list_find(struct loc_network_list* list,
 		struct loc_network* network, int* found) {
 	off_t lo = 0;
 	off_t hi = list->size - 1;
+	int result;
 
-	*found = 0;
+	// Since we are working on an ordered list, there is often a good chance that
+	// the network we are looking for is at the end or has to go to the end.
+	if (hi >= 0) {
+		result = loc_network_cmp(network, list->elements[hi]);
+
+		// Match, so we are done
+		if (result == 0) {
+			*found = 1;
+
+			return hi;
+
+		// This needs to be added after the last one
+		} else if (result > 0) {
+			*found = 0;
+
+			return hi + 1;
+		}
+	}
 
 #ifdef ENABLE_DEBUG
 	// Save start time
@@ -151,7 +169,7 @@ static off_t loc_network_list_find(struct loc_network_list* list,
 		i = (lo + hi) / 2;
 
 		// Check if this is a match
-		int result = loc_network_cmp(network, list->elements[i]);
+		result = loc_network_cmp(network, list->elements[i]);
 
 		if (result == 0) {
 			*found = 1;
@@ -173,6 +191,8 @@ static off_t loc_network_list_find(struct loc_network_list* list,
 			hi = i - 1;
 	}
 
+	*found = 0;
+
 #ifdef ENABLE_DEBUG
 	clock_t end = clock();
 
-- 
2.20.1

From d42f34ce9b0aa2382ae9b852882c62b34d367cfe Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 14:41:39 +0000
Subject: [PATCH 098/111] network-list: Set elements pointer to NULL so that we
 know it is empty

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network-list.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/network-list.c b/src/network-list.c
index c86ad07..2c4edb3 100644
--- a/src/network-list.c
+++ b/src/network-list.c
@@ -104,6 +104,7 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
 		loc_network_unref(list->elements[i]);
 
 	free(list->elements);
+	list->elements = NULL;
 	list->elements_size = 0;
 
 	list->size = 0;
-- 
2.20.1

From 673e03f7bfc807248a769cb00ec80f0fa2af7a25 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 14:42:26 +0000
Subject: [PATCH 099/111] network-list: Do not half list when popping the first
 element

The list was unfortunately halved in size every time an element
was taken from it, which was great for performance, but shortened
the result substantially.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network-list.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/network-list.c b/src/network-list.c
index 2c4edb3..f4a9d05 100644
--- a/src/network-list.c
+++ b/src/network-list.c
@@ -261,10 +261,13 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
 	struct loc_network* network = list->elements[0];
 
 	// Move all elements to the top of the stack
-	for (unsigned int i = 0; i < --list->size; i++) {
+	for (unsigned int i = 0; i < list->size - 1; i++) {
 		list->elements[i] = list->elements[i+1];
 	}
 
+	// The list is shorter now
+	--list->size;
+
 	DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
 
 	return network;
-- 
2.20.1

From 9446c7538ab8b27486d74f6dc0dac8bb5c1bbe92 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 14:43:58 +0000
Subject: [PATCH 100/111] network-list: Show index when listing networks

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network-list.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/network-list.c b/src/network-list.c
index f4a9d05..cf9459d 100644
--- a/src/network-list.c
+++ b/src/network-list.c
@@ -119,7 +119,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
 
 		s = loc_network_str(network);
 
-		INFO(list->ctx, "%s\n", s);
+		INFO(list->ctx, "%4d: %s\n", i, s);
 		free(s);
 	}
 }
-- 
2.20.1

From dc31666be416dbb3bd2bc04127104a3f75b42d5c Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 14:44:23 +0000
Subject: [PATCH 101/111] network-list: Remove useless comment

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network-list.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/network-list.c b/src/network-list.c
index cf9459d..bd3d64e 100644
--- a/src/network-list.c
+++ b/src/network-list.c
@@ -132,8 +132,6 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
 	return loc_network_ref(list->elements[index]);
 }
 
-//MOVE FUNCTION GOES HERE
-
 static off_t loc_network_list_find(struct loc_network_list* list,
 		struct loc_network* network, int* found) {
 	off_t lo = 0;
-- 
2.20.1

From 82fa4c92c88cc172953198637bccc375c4d25d20 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 14:44:56 +0000
Subject: [PATCH 102/111] networks: Add tests for overlaps function

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/test-network.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/test-network.c b/src/test-network.c
index 339743d..dde13f1 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -192,6 +192,16 @@ int main(int argc, char** argv) {
 		exit(EXIT_FAILURE);
 	}
 
+	if (!loc_network_overlaps(network1, subnet1)) {
+		fprintf(stderr, "Network1 does not seem to contain subnet1\n");
+		exit(EXIT_FAILURE);
+	}
+
+	if (!loc_network_overlaps(network1, subnet2)) {
+		fprintf(stderr, "Network1 does not seem to contain subnet2\n");
+		exit(EXIT_FAILURE);
+	}
+
 	loc_network_unref(subnet1);
 	loc_network_unref(subnet2);
 
-- 
2.20.1

From 0a39d5499ae0c343a6600ea30de7d95e384ef911 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 15:11:21 +0000
Subject: [PATCH 103/111] networks: Remove comparing family

Everything is encoded in IPv6 anyways...

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/src/network.c b/src/network.c
index 394bafc..66b460f 100644
--- a/src/network.c
+++ b/src/network.c
@@ -442,12 +442,6 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
 }
 
 LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
-	// Compare family
-	if (self->family > other->family)
-		return 1;
-	else if (self->family < other->family)
-		return -1;
-
 	// Compare address
 	int r = in6_addr_cmp(&self->first_address, &other->first_address);
 	if (r)
@@ -482,10 +476,6 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
 }
 
 LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
-	// Check family
-	if (self->family != other->family)
-		return 0;
-
 	// The prefix must be smaller (this avoids the more complex comparisons later)
 	if (self->prefix > other->prefix)
 		return 0;
@@ -601,13 +591,6 @@ ERROR:
 
 static int __loc_network_exclude_to_list(struct loc_network* self,
 		struct loc_network* other, struct loc_network_list* list) {
-	// Family must match
-	if (self->family != other->family) {
-		DEBUG(self->ctx, "Family mismatch\n");
-
-		return 1;
-	}
-
 	// Other must be a subnet of self
 	if (!loc_network_is_subnet(self, other)) {
 		DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
-- 
2.20.1

From abf559267696061a0c9723d8f8e09c9d1aa8fb75 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 15:13:08 +0000
Subject: [PATCH 104/111] network: Do not execute with an error when the
 excluded result will be empty

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/network.c b/src/network.c
index 66b460f..9aa802f 100644
--- a/src/network.c
+++ b/src/network.c
@@ -595,14 +595,16 @@ static int __loc_network_exclude_to_list(struct loc_network* self,
 	if (!loc_network_is_subnet(self, other)) {
 		DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
 
-		return 1;
+		// Exit silently
+		return 0;
 	}
 
 	// We cannot perform this operation if both networks equal
 	if (loc_network_cmp(self, other) == 0) {
 		DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
 
-		return 1;
+		// Exit silently
+		return 0;
 	}
 
 	return __loc_network_exclude(self, other, list);
-- 
2.20.1

From 4fc034e20fdc36267f6ba18e27776fed747fe983 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 15:14:31 +0000
Subject: [PATCH 105/111] network: Add more excluded networks straight to the
 to_check list

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/network.c b/src/network.c
index 9aa802f..19c387d 100644
--- a/src/network.c
+++ b/src/network.c
@@ -705,11 +705,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
 			if (loc_network_overlaps(subnet_to_check, subnet)) {
 				passed = 0;
 
-				struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet);
-				if (excluded) {
-					loc_network_list_merge(to_check, excluded);
-					loc_network_list_unref(excluded);
-				}
+				__loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
 
 				loc_network_unref(subnet);
 				break;
-- 
2.20.1

From 4de8ff8e5435225d3375a168054490d2a66b1baf Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 15:15:33 +0000
Subject: [PATCH 106/111] network: Call subnet function with the correct order
 of arguments

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/network.c b/src/network.c
index 19c387d..a96ce6d 100644
--- a/src/network.c
+++ b/src/network.c
@@ -695,14 +695,14 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
 			subnet = loc_network_list_get(list, i);
 
 			// Drop this subnet if is a subnet of another subnet
-			if (loc_network_is_subnet(subnet_to_check, subnet)) {
+			if (loc_network_is_subnet(subnet, subnet_to_check)) {
 				passed = 0;
 				loc_network_unref(subnet);
 				break;
 			}
 
 			// Break it down if it overlaps
-			if (loc_network_overlaps(subnet_to_check, subnet)) {
+			if (loc_network_overlaps(subnet, subnet_to_check)) {
 				passed = 0;
 
 				__loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
-- 
2.20.1

From 058e7800e56150bcf0fb8dceaae6fbab1ed239e4 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 15:16:06 +0000
Subject: [PATCH 107/111] network: Massively improve performance on exclude

When we check the result for any overlaps, we can cut this short
by walking through both lists from start to end and remember the
last network that we checked.

The next one will by definition be strictly greater and therefore
we do not need to check anything before this any more.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/network.c b/src/network.c
index a96ce6d..10fa997 100644
--- a/src/network.c
+++ b/src/network.c
@@ -679,8 +679,10 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
 		return NULL;
 	}
 
+	off_t smallest_subnet = 0;
+
 	while (!loc_network_list_empty(to_check)) {
-		struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
+		struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check);
 
 		// Check whether the subnet to check is part of the input list
 		if (loc_network_list_contains(list, subnet_to_check)) {
@@ -691,7 +693,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
 		// Marks whether this subnet passed all checks
 		int passed = 1;
 
-		for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
+		for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) {
 			subnet = loc_network_list_get(list, i);
 
 			// Drop this subnet if is a subnet of another subnet
@@ -711,6 +713,19 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
 				break;
 			}
 
+			// If the subnet is strictly greater, we do not need to continue the search
+			r = loc_network_cmp(subnet, subnet_to_check);
+			if (r > 0) {
+				loc_network_unref(subnet);
+				break;
+
+			// If it is strictly smaller, we can continue the search from here next
+			// time because all networks that are to be checked can only be larger
+			// than this one.
+			} else if (r < 0) {
+				smallest_subnet = i;
+			}
+
 			loc_network_unref(subnet);
 		}
 
-- 
2.20.1

From 9caf6cf5f0fd5eb56ad1678910dbc8017f48863c Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 15:17:21 +0000
Subject: [PATCH 108/111] network: Remove deprecated sort call

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/network.c b/src/network.c
index 10fa997..4d7ac79 100644
--- a/src/network.c
+++ b/src/network.c
@@ -738,9 +738,6 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
 
 	loc_network_list_unref(to_check);
 
-	// Sort the result
-	loc_network_list_sort(subnets);
-
 	return subnets;
 }
 
-- 
2.20.1

From 8ebf1d3912020d5c0ed98bb32a0640952bd6447c Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 15:17:42 +0000
Subject: [PATCH 109/111] network-list: Include network header

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/loc/network-list.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/loc/network-list.h b/src/loc/network-list.h
index 21c7402..bee21c4 100644
--- a/src/loc/network-list.h
+++ b/src/loc/network-list.h
@@ -17,6 +17,8 @@
 #ifndef LIBLOC_NETWORK_LIST_H
 #define LIBLOC_NETWORK_LIST_H
 
+#include <loc/network.h>
+
 struct loc_network_list;
 int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
 struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
-- 
2.20.1

From fff093b6827a31ff84a6d5150bfede140b696d25 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 15:24:43 +0000
Subject: [PATCH 110/111] network-list: Use clear function to tidy up

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 src/network-list.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/network-list.c b/src/network-list.c
index bd3d64e..f3458b4 100644
--- a/src/network-list.c
+++ b/src/network-list.c
@@ -70,8 +70,8 @@ LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list
 static void loc_network_list_free(struct loc_network_list* list) {
 	DEBUG(list->ctx, "Releasing network list at %p\n", list);
 
-	for (unsigned int i = 0; i < list->size; i++)
-		loc_network_unref(list->elements[i]);
+	// Remove all content
+	loc_network_list_clear(list);
 
 	loc_unref(list->ctx);
 	free(list);
-- 
2.20.1

From adbec5c0317a1c007a897fe3f63f0f86ff448124 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 25 Nov 2020 20:00:46 +0000
Subject: [PATCH 111/111] configure: Bump version to 0.9.5

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 2364dfd..012d8ca 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,6 +1,6 @@
 AC_PREREQ(2.60)
 AC_INIT([libloc],
-        [0.9.4],
+        [0.9.5],
         [location@lists.ipfire.org],
         [libloc],
         [https://location.ipfire.org/])
-- 
2.20.1