--- /dev/null
+From ee6ea3986dc80183157f67275dc9f28231b5d5b2 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 24 Sep 2020 10:17:58 +0000
+Subject: [PATCH 01/70] Revert "importer: Purge any redundant entries"
+
+This reverts commit c2cc55d5a6875c3838f060032eaed89dcfb92ef6.
+
+The query stalls the database and therefore the automatic
+scripts are no longer able to generate a new version of the
+database.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 22 +---------------------
+ 1 file changed, 1 insertion(+), 21 deletions(-)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index 1467923..e3a07a0 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -374,27 +374,7 @@ class CLI(object):
+ INSERT INTO autnums(number, name)
+ SELECT _autnums.number, _organizations.name FROM _autnums
+ JOIN _organizations ON _autnums.organization = _organizations.handle
+- ON CONFLICT (number) DO UPDATE SET name = excluded.name
+- """)
+-
+- self.db.execute("""
+- --- Purge any redundant entries
+- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
+- AS
+- SELECT network FROM networks candidates
+- WHERE EXISTS (
+- SELECT FROM networks
+- WHERE
+- networks.network << candidates.network
+- AND
+- networks.country = candidates.country
+- );
+-
+- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
+-
+- DELETE FROM networks WHERE EXISTS (
+- SELECT FROM _garbage WHERE networks.network = _garbage.network
+- );
++ ON CONFLICT (number) DO UPDATE SET name = excluded.name;
+ """)
+
+ # Download all extended sources
+--
+2.20.1
+
+From 92f6abf4e272672bb0a71cfe991261b95ebe2fef Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 24 Sep 2020 10:18:58 +0000
+Subject: [PATCH 02/70] Revert "importer: Import raw sources for inetnum's
+ again"
+
+This reverts commit 64e95fa903edec8b4e4e59830b395e2e4a411853.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/importer.py | 14 ++++----
+ src/python/location-importer.in | 63 ---------------------------------
+ 2 files changed, 7 insertions(+), 70 deletions(-)
+
+diff --git a/src/python/importer.py b/src/python/importer.py
+index f19db4b..de20f37 100644
+--- a/src/python/importer.py
++++ b/src/python/importer.py
+@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
+ "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
+
+ # Asia Pacific Network Information Centre
+- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+ #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
+ #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
+ "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
+@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
+ # XXX ???
+
+ # Réseaux IP Européens
+- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
+@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
+
+ EXTENDED_SOURCES = (
+ # African Network Information Centre
+- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
++ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+
+ # Asia Pacific Network Information Centre
+- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
++ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+
+ # American Registry for Internet Numbers
+ "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
+@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
+ "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
+
+ # Réseaux IP Européens
+- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
++ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+ )
+
+ class Downloader(object):
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index e3a07a0..77952f2 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -393,10 +393,6 @@ class CLI(object):
+ if line.startswith("aut-num:"):
+ return self._parse_autnum_block(block)
+
+- # inetnum
+- if line.startswith("inet6num:") or line.startswith("inetnum:"):
+- return self._parse_inetnum_block(block)
+-
+ # organisation
+ elif line.startswith("organisation:"):
+ return self._parse_org_block(block)
+@@ -426,65 +422,6 @@ class CLI(object):
+ autnum.get("asn"), autnum.get("org"),
+ )
+
+- def _parse_inetnum_block(self, block):
+- logging.debug("Parsing inetnum block:")
+-
+- inetnum = {}
+- for line in block:
+- logging.debug(line)
+-
+- # Split line
+- key, val = split_line(line)
+-
+- if key == "inetnum":
+- start_address, delim, end_address = val.partition("-")
+-
+- # Strip any excess space
+- start_address, end_address = start_address.rstrip(), end_address.strip()
+-
+- # Convert to IP address
+- try:
+- start_address = ipaddress.ip_address(start_address)
+- end_address = ipaddress.ip_address(end_address)
+- except ValueError:
+- logging.warning("Could not parse line: %s" % line)
+- return
+-
+- # Set prefix to default
+- prefix = 32
+-
+- # Count number of addresses in this subnet
+- num_addresses = int(end_address) - int(start_address)
+- if num_addresses:
+- prefix -= math.log(num_addresses, 2)
+-
+- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
+-
+- elif key == "inet6num":
+- inetnum[key] = val
+-
+- elif key == "country":
+- if val == "UNITED STATES":
+- val = "US"
+-
+- inetnum[key] = val.upper()
+-
+- # Skip empty objects
+- if not inetnum:
+- return
+-
+- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+-
+- # Bail out in case we have processed a non-public IP network
+- if network.is_private:
+- logging.warning("Skipping non-globally routable network: %s" % network)
+- return
+-
+- self.db.execute("INSERT INTO networks(network, country) \
+- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+- "%s" % network, inetnum.get("country"),
+- )
+-
+ def _parse_org_block(self, block):
+ org = {}
+ for line in block:
+--
+2.20.1
+
+From f532841e9197ce2f40aad8c086d786b2cb783a54 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Mon, 12 Oct 2020 20:53:31 +0000
+Subject: [PATCH 03/70] Revert "Revert "importer: Import raw sources for
+ inetnum's again""
+
+This reverts commit 92f6abf4e272672bb0a71cfe991261b95ebe2fef.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/importer.py | 14 ++++----
+ src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
+ 2 files changed, 70 insertions(+), 7 deletions(-)
+
+diff --git a/src/python/importer.py b/src/python/importer.py
+index de20f37..f19db4b 100644
+--- a/src/python/importer.py
++++ b/src/python/importer.py
+@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
+ "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
+
+ # Asia Pacific Network Information Centre
+- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
++ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
++ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+ #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
+ #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
+ "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
+@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
+ # XXX ???
+
+ # Réseaux IP Européens
+- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
+@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
+
+ EXTENDED_SOURCES = (
+ # African Network Information Centre
+- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
++ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+
+ # Asia Pacific Network Information Centre
+- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
++ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+
+ # American Registry for Internet Numbers
+ "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
+@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
+ "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
+
+ # Réseaux IP Européens
+- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
++ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+ )
+
+ class Downloader(object):
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index 77952f2..e3a07a0 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -393,6 +393,10 @@ class CLI(object):
+ if line.startswith("aut-num:"):
+ return self._parse_autnum_block(block)
+
++ # inetnum
++ if line.startswith("inet6num:") or line.startswith("inetnum:"):
++ return self._parse_inetnum_block(block)
++
+ # organisation
+ elif line.startswith("organisation:"):
+ return self._parse_org_block(block)
+@@ -422,6 +426,65 @@ class CLI(object):
+ autnum.get("asn"), autnum.get("org"),
+ )
+
++ def _parse_inetnum_block(self, block):
++ logging.debug("Parsing inetnum block:")
++
++ inetnum = {}
++ for line in block:
++ logging.debug(line)
++
++ # Split line
++ key, val = split_line(line)
++
++ if key == "inetnum":
++ start_address, delim, end_address = val.partition("-")
++
++ # Strip any excess space
++ start_address, end_address = start_address.rstrip(), end_address.strip()
++
++ # Convert to IP address
++ try:
++ start_address = ipaddress.ip_address(start_address)
++ end_address = ipaddress.ip_address(end_address)
++ except ValueError:
++ logging.warning("Could not parse line: %s" % line)
++ return
++
++ # Set prefix to default
++ prefix = 32
++
++ # Count number of addresses in this subnet
++ num_addresses = int(end_address) - int(start_address)
++ if num_addresses:
++ prefix -= math.log(num_addresses, 2)
++
++ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
++
++ elif key == "inet6num":
++ inetnum[key] = val
++
++ elif key == "country":
++ if val == "UNITED STATES":
++ val = "US"
++
++ inetnum[key] = val.upper()
++
++ # Skip empty objects
++ if not inetnum:
++ return
++
++ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
++
++ # Bail out in case we have processed a non-public IP network
++ if network.is_private:
++ logging.warning("Skipping non-globally routable network: %s" % network)
++ return
++
++ self.db.execute("INSERT INTO networks(network, country) \
++ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
++ "%s" % network, inetnum.get("country"),
++ )
++
+ def _parse_org_block(self, block):
+ org = {}
+ for line in block:
+--
+2.20.1
+
+From a36bc686865fc87ea386fd90b389338bdcb80cbc Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Mon, 12 Oct 2020 20:53:32 +0000
+Subject: [PATCH 04/70] location-importer.in: only import relevant data from
+ AFRINIC, APNIC and RIPE
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+In contrast to ARIN and LACNIC, we are able to process more detailled
+feeds from those RIRs, avoiding storage of obviously unnecessary data.
+
+Thanks to various SQL optimisations, doing so now takes less time than
+the first version of this did.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
+ 1 file changed, 87 insertions(+), 2 deletions(-)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index e3a07a0..093f325 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -165,6 +165,7 @@ class CLI(object):
+ -- networks
+ CREATE TABLE IF NOT EXISTS networks(network inet, country text);
+ CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
++ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
+ CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
+
+ -- overrides
+@@ -363,6 +364,16 @@ class CLI(object):
+ CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
+ ON COMMIT DROP;
+ CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
++
++ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
++ ON COMMIT DROP;
++ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
++ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
++ """)
++
++ # Remove all previously imported content
++ self.db.execute("""
++ TRUNCATE TABLE networks;
+ """)
+
+ for source in location.importer.WHOIS_SOURCES:
+@@ -370,6 +381,67 @@ class CLI(object):
+ for block in f:
+ self._parse_block(block)
+
++ # Process all parsed networks from every RIR we happen to have access to,
++ # insert the largest network chunks into the networks table immediately...
++ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
++
++ for family in (row.family for row in families):
++ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
++
++ self.db.execute("INSERT INTO networks(network, country) \
++ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
++
++ # ... determine any other prefixes for this network family, ...
++ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
++ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
++
++ # ... and insert networks with this prefix in case they provide additional
++ # information (i. e. subnet of a larger chunk with a different country)
++ for prefix in (row.prefix for row in prefixes):
++ self.db.execute("""
++ WITH candidates AS (
++ SELECT
++ _rirdata.network,
++ _rirdata.country
++ FROM
++ _rirdata
++ WHERE
++ family(_rirdata.network) = %s
++ AND
++ masklen(_rirdata.network) = %s
++ ),
++ filtered AS (
++ SELECT
++ DISTINCT ON (c.network)
++ c.network,
++ c.country,
++ masklen(networks.network),
++ networks.country AS parent_country
++ FROM
++ candidates c
++ LEFT JOIN
++ networks
++ ON
++ c.network << networks.network
++ ORDER BY
++ c.network,
++ masklen(networks.network) DESC NULLS LAST
++ )
++ INSERT INTO
++ networks(network, country)
++ SELECT
++ network,
++ country
++ FROM
++ filtered
++ WHERE
++ parent_country IS NULL
++ OR
++ country <> parent_country
++ ON CONFLICT DO NOTHING""",
++ family, prefix,
++ )
++
+ self.db.execute("""
+ INSERT INTO autnums(number, name)
+ SELECT _autnums.number, _organizations.name FROM _autnums
+@@ -470,17 +542,30 @@ class CLI(object):
+ inetnum[key] = val.upper()
+
+ # Skip empty objects
+- if not inetnum:
++ if not inetnum or not "country" in inetnum:
+ return
+
+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
++ # Bail out in case we have processed a network covering the entire IP range, which
++ # is necessary to work around faulty (?) IPv6 network processing
++ if network.prefixlen == 0:
++ logging.warning("Skipping network covering the entire IP adress range: %s" % network)
++ return
++
++ # Bail out in case we have processed a network whose prefix length indicates it is
++ # not globally routable (we have decided not to process them at the moment, as they
++ # significantly enlarge our database without providing very helpful additional information)
++ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
++ logging.info("Skipping network too small to be publicly announced: %s" % network)
++ return
++
+ # Bail out in case we have processed a non-public IP network
+ if network.is_private:
+ logging.warning("Skipping non-globally routable network: %s" % network)
+ return
+
+- self.db.execute("INSERT INTO networks(network, country) \
++ self.db.execute("INSERT INTO _rirdata(network, country) \
+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+ "%s" % network, inetnum.get("country"),
+ )
+--
+2.20.1
+
+From 2373de384f10f5573bbd7570f5522545df70c0e3 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri, 16 Oct 2020 12:24:58 +0000
+Subject: [PATCH 05/70] location-importer: Include all overridden networks
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index 093f325..d249a35 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -240,6 +240,8 @@ class CLI(object):
+ SELECT network FROM announcements
+ UNION
+ SELECT network FROM networks
++ UNION
++ SELECT network FROM network_overrides
+ ORDER BY network
+ )
+
+--
+2.20.1
+
+From 13f67f285856e8eabfeff2daf1be3aeaa36a82cc Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri, 16 Oct 2020 12:26:38 +0000
+Subject: [PATCH 06/70] Revert "location-importer.in: only import relevant data
+ from AFRINIC, APNIC and RIPE"
+
+This reverts commit a36bc686865fc87ea386fd90b389338bdcb80cbc.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 89 +--------------------------------
+ 1 file changed, 2 insertions(+), 87 deletions(-)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index d249a35..b220eaf 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -165,7 +165,6 @@ class CLI(object):
+ -- networks
+ CREATE TABLE IF NOT EXISTS networks(network inet, country text);
+ CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
+- CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
+ CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
+
+ -- overrides
+@@ -366,16 +365,6 @@ class CLI(object):
+ CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
+ ON COMMIT DROP;
+ CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
+-
+- CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
+- ON COMMIT DROP;
+- CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
+- CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
+- """)
+-
+- # Remove all previously imported content
+- self.db.execute("""
+- TRUNCATE TABLE networks;
+ """)
+
+ for source in location.importer.WHOIS_SOURCES:
+@@ -383,67 +372,6 @@ class CLI(object):
+ for block in f:
+ self._parse_block(block)
+
+- # Process all parsed networks from every RIR we happen to have access to,
+- # insert the largest network chunks into the networks table immediately...
+- families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
+-
+- for family in (row.family for row in families):
+- smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
+-
+- self.db.execute("INSERT INTO networks(network, country) \
+- SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
+-
+- # ... determine any other prefixes for this network family, ...
+- prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
+- WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
+-
+- # ... and insert networks with this prefix in case they provide additional
+- # information (i. e. subnet of a larger chunk with a different country)
+- for prefix in (row.prefix for row in prefixes):
+- self.db.execute("""
+- WITH candidates AS (
+- SELECT
+- _rirdata.network,
+- _rirdata.country
+- FROM
+- _rirdata
+- WHERE
+- family(_rirdata.network) = %s
+- AND
+- masklen(_rirdata.network) = %s
+- ),
+- filtered AS (
+- SELECT
+- DISTINCT ON (c.network)
+- c.network,
+- c.country,
+- masklen(networks.network),
+- networks.country AS parent_country
+- FROM
+- candidates c
+- LEFT JOIN
+- networks
+- ON
+- c.network << networks.network
+- ORDER BY
+- c.network,
+- masklen(networks.network) DESC NULLS LAST
+- )
+- INSERT INTO
+- networks(network, country)
+- SELECT
+- network,
+- country
+- FROM
+- filtered
+- WHERE
+- parent_country IS NULL
+- OR
+- country <> parent_country
+- ON CONFLICT DO NOTHING""",
+- family, prefix,
+- )
+-
+ self.db.execute("""
+ INSERT INTO autnums(number, name)
+ SELECT _autnums.number, _organizations.name FROM _autnums
+@@ -544,30 +472,17 @@ class CLI(object):
+ inetnum[key] = val.upper()
+
+ # Skip empty objects
+- if not inetnum or not "country" in inetnum:
++ if not inetnum:
+ return
+
+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
+- # Bail out in case we have processed a network covering the entire IP range, which
+- # is necessary to work around faulty (?) IPv6 network processing
+- if network.prefixlen == 0:
+- logging.warning("Skipping network covering the entire IP adress range: %s" % network)
+- return
+-
+- # Bail out in case we have processed a network whose prefix length indicates it is
+- # not globally routable (we have decided not to process them at the moment, as they
+- # significantly enlarge our database without providing very helpful additional information)
+- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
+- logging.info("Skipping network too small to be publicly announced: %s" % network)
+- return
+-
+ # Bail out in case we have processed a non-public IP network
+ if network.is_private:
+ logging.warning("Skipping non-globally routable network: %s" % network)
+ return
+
+- self.db.execute("INSERT INTO _rirdata(network, country) \
++ self.db.execute("INSERT INTO networks(network, country) \
+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+ "%s" % network, inetnum.get("country"),
+ )
+--
+2.20.1
+
+From 44341478233115b26bb27fdb24da5b0a1eedb173 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri, 16 Oct 2020 12:26:43 +0000
+Subject: [PATCH 07/70] Revert "Revert "Revert "importer: Import raw sources
+ for inetnum's again"""
+
+This reverts commit f532841e9197ce2f40aad8c086d786b2cb783a54.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/importer.py | 14 ++++----
+ src/python/location-importer.in | 63 ---------------------------------
+ 2 files changed, 7 insertions(+), 70 deletions(-)
+
+diff --git a/src/python/importer.py b/src/python/importer.py
+index f19db4b..de20f37 100644
+--- a/src/python/importer.py
++++ b/src/python/importer.py
+@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
+ "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
+
+ # Asia Pacific Network Information Centre
+- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+ #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
+ #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
+ "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
+@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
+ # XXX ???
+
+ # Réseaux IP Européens
+- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
+@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
+
+ EXTENDED_SOURCES = (
+ # African Network Information Centre
+- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
++ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+
+ # Asia Pacific Network Information Centre
+- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
++ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+
+ # American Registry for Internet Numbers
+ "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
+@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
+ "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
+
+ # Réseaux IP Européens
+- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
++ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+ )
+
+ class Downloader(object):
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index b220eaf..e87d378 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -395,10 +395,6 @@ class CLI(object):
+ if line.startswith("aut-num:"):
+ return self._parse_autnum_block(block)
+
+- # inetnum
+- if line.startswith("inet6num:") or line.startswith("inetnum:"):
+- return self._parse_inetnum_block(block)
+-
+ # organisation
+ elif line.startswith("organisation:"):
+ return self._parse_org_block(block)
+@@ -428,65 +424,6 @@ class CLI(object):
+ autnum.get("asn"), autnum.get("org"),
+ )
+
+- def _parse_inetnum_block(self, block):
+- logging.debug("Parsing inetnum block:")
+-
+- inetnum = {}
+- for line in block:
+- logging.debug(line)
+-
+- # Split line
+- key, val = split_line(line)
+-
+- if key == "inetnum":
+- start_address, delim, end_address = val.partition("-")
+-
+- # Strip any excess space
+- start_address, end_address = start_address.rstrip(), end_address.strip()
+-
+- # Convert to IP address
+- try:
+- start_address = ipaddress.ip_address(start_address)
+- end_address = ipaddress.ip_address(end_address)
+- except ValueError:
+- logging.warning("Could not parse line: %s" % line)
+- return
+-
+- # Set prefix to default
+- prefix = 32
+-
+- # Count number of addresses in this subnet
+- num_addresses = int(end_address) - int(start_address)
+- if num_addresses:
+- prefix -= math.log(num_addresses, 2)
+-
+- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
+-
+- elif key == "inet6num":
+- inetnum[key] = val
+-
+- elif key == "country":
+- if val == "UNITED STATES":
+- val = "US"
+-
+- inetnum[key] = val.upper()
+-
+- # Skip empty objects
+- if not inetnum:
+- return
+-
+- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+-
+- # Bail out in case we have processed a non-public IP network
+- if network.is_private:
+- logging.warning("Skipping non-globally routable network: %s" % network)
+- return
+-
+- self.db.execute("INSERT INTO networks(network, country) \
+- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+- "%s" % network, inetnum.get("country"),
+- )
+-
+ def _parse_org_block(self, block):
+ org = {}
+ for line in block:
+--
+2.20.1
+
+From a7d3a7a0565a0e09d3442e5829a0f30f016993b9 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 20 Oct 2020 20:44:43 +0000
+Subject: [PATCH 08/70] as: Fix dereferencing NULL pointer when setting AS name
+
+Reported-by: Gisle Vanem <gisle.vanem@gmail.com>
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/as.c | 8 +++++++-
+ 1 file changed, 7 insertions(+), 1 deletion(-)
+
+diff --git a/src/as.c b/src/as.c
+index e1fbb01..8421ac8 100644
+--- a/src/as.c
++++ b/src/as.c
+@@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
+ }
+
+ LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) {
+- as->name = strdup(name);
++ if (as->name)
++ free(as->name);
++
++ if (name)
++ as->name = strdup(name);
++ else
++ as->name = NULL;
+
+ return 0;
+ }
+--
+2.20.1
+
+From ddb326ad38a7c7202315dd2c6f938313db04ee22 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 21 Oct 2020 09:18:08 +0000
+Subject: [PATCH 09/70] as: Do not attempt to match name when it wasn't set
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/as.c | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/src/as.c b/src/as.c
+index 8421ac8..757bf3d 100644
+--- a/src/as.c
++++ b/src/as.c
+@@ -145,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
+ if (!string)
+ return 1;
+
++ // Cannot match anything when name is not set
++ if (!as->name)
++ return 1;
++
+ // Search if string is in name
+ if (strcasestr(as->name, string) != NULL)
+ return 1;
+--
+2.20.1
+
+From d226ad2d97cbcd42ce807d9308569b1b9c5d4e2f Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 21 Oct 2020 09:28:39 +0000
+Subject: [PATCH 10/70] writer: Free array with pointer to ASes, too
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/writer.c | 7 +++++--
+ 1 file changed, 5 insertions(+), 2 deletions(-)
+
+diff --git a/src/writer.c b/src/writer.c
+index 5939cff..160650f 100644
+--- a/src/writer.c
++++ b/src/writer.c
+@@ -147,8 +147,11 @@ static void loc_writer_free(struct loc_writer* writer) {
+ EVP_PKEY_free(writer->private_key2);
+
+ // Unref all AS
+- for (unsigned int i = 0; i < writer->as_count; i++) {
+- loc_as_unref(writer->as[i]);
++ if (writer->as) {
++ for (unsigned int i = 0; i < writer->as_count; i++) {
++ loc_as_unref(writer->as[i]);
++ }
++ free(writer->as);
+ }
+
+ // Release network tree
+--
+2.20.1
+
+From d89a7d62772048ae1bd18d03f69df46b7e1a3d3c Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 21 Oct 2020 09:31:29 +0000
+Subject: [PATCH 11/70] writer: Free countries when the writer is being
+ destroyed
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/writer.c | 8 ++++++++
+ 1 file changed, 8 insertions(+)
+
+diff --git a/src/writer.c b/src/writer.c
+index 160650f..2f09b56 100644
+--- a/src/writer.c
++++ b/src/writer.c
+@@ -154,6 +154,14 @@ static void loc_writer_free(struct loc_writer* writer) {
+ free(writer->as);
+ }
+
++ // Unref all countries
++ if (writer->countries) {
++ for (unsigned int i = 0; i < writer->countries_count; i++) {
++ loc_country_unref(writer->countries[i]);
++ }
++ free(writer->countries);
++ }
++
+ // Release network tree
+ if (writer->networks)
+ loc_network_tree_unref(writer->networks);
+--
+2.20.1
+
+From 0f1aedbc68e3945770c93e0ebd83eed0f555d6f0 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 21 Oct 2020 13:19:44 +0000
+Subject: [PATCH 12/70] tests: Try adding an invalid network
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/test-network.c | 8 ++++++++
+ 1 file changed, 8 insertions(+)
+
+diff --git a/src/test-network.c b/src/test-network.c
+index d38f13d..e908b57 100644
+--- a/src/test-network.c
++++ b/src/test-network.c
+@@ -160,6 +160,14 @@ int main(int argc, char** argv) {
+ // Set ASN
+ loc_network_set_asn(network4, 1024);
+
++ // Try adding an invalid network
++ struct loc_network* network;
++ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
++ if (err != -EINVAL) {
++ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
++ exit(EXIT_FAILURE);
++ }
++
+ FILE* f = tmpfile();
+ if (!f) {
+ fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
+--
+2.20.1
+
+From 13ad6e695f9ffc7847b3afe3e9cbcea8fb3a443f Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 21 Oct 2020 13:36:35 +0000
+Subject: [PATCH 13/70] networks: Improve parsing IP addresses
+
+loc_network_new_from_string() seem to have had some unexpected
+behaviour for invalid inputs.
+
+The function has been tidied up slightly and returns as soon as
+some invalid input was detected.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/network.c | 52 +++++++++++++++++++++++++++++++---------------
+ src/test-network.c | 7 +++++++
+ 2 files changed, 42 insertions(+), 17 deletions(-)
+
+diff --git a/src/network.c b/src/network.c
+index 366caa2..c112a41 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -160,9 +160,10 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
+ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
+ const char* address_string) {
+ struct in6_addr first_address;
+- unsigned int prefix = 0;
+ char* prefix_string;
+- int r = 1;
++ int r = -EINVAL;
++
++ DEBUG(ctx, "Attempting to parse network %s\n", address_string);
+
+ // Make a copy of the string to work on it
+ char* buffer = strdup(address_string);
+@@ -171,29 +172,46 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
+ // Split address and prefix
+ address_string = strsep(&prefix_string, "/");
+
+- // Did we find a prefix?
+- if (prefix_string) {
+- // Convert prefix to integer
+- prefix = strtol(prefix_string, NULL, 10);
++ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
+
+- if (prefix) {
+- // Parse the address
+- r = loc_parse_address(ctx, address_string, &first_address);
++ // We need to have a prefix
++ if (!prefix_string) {
++ DEBUG(ctx, "No prefix set\n");
++ goto FAIL;
++ }
+
+- // Map the prefix to IPv6 if needed
+- if (IN6_IS_ADDR_V4MAPPED(&first_address))
+- prefix += 96;
+- }
++ // Convert prefix to integer
++ unsigned int prefix = strtol(prefix_string, NULL, 10);
++
++ // End if the prefix was invalid
++ if (!prefix) {
++ DEBUG(ctx, "The prefix is zero or not a number\n");
++ goto FAIL;
++ }
++
++ // Parse the address
++ r = loc_parse_address(ctx, address_string, &first_address);
++ if (r) {
++ DEBUG(ctx, "The address could not be parsed\n");
++ goto FAIL;
+ }
+
++ // Map the prefix to IPv6 if needed
++ if (IN6_IS_ADDR_V4MAPPED(&first_address))
++ prefix += 96;
++
++FAIL:
+ // Free temporary buffer
+ free(buffer);
+
+- if (r == 0) {
+- r = loc_network_new(ctx, network, &first_address, prefix);
+- }
++ // Exit if the parsing was unsuccessful
++ if (r)
++ return r;
++
++ DEBUG(ctx, "GOT HERE\n");
+
+- return r;
++ // Create a new network
++ return loc_network_new(ctx, network, &first_address, prefix);
+ }
+
+ LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
+diff --git a/src/test-network.c b/src/test-network.c
+index e908b57..85eca00 100644
+--- a/src/test-network.c
++++ b/src/test-network.c
+@@ -168,6 +168,13 @@ int main(int argc, char** argv) {
+ exit(EXIT_FAILURE);
+ }
+
++ // Try adding a single address
++ err = loc_writer_add_network(writer, &network, "2001:db8::");
++ if (err != -EINVAL) {
++ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
++ exit(EXIT_FAILURE);
++ }
++
+ FILE* f = tmpfile();
+ if (!f) {
+ fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
+--
+2.20.1
+
+From 6a467e9345bb5a3d37911c9aaac30019eaa4492b Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 21 Oct 2020 13:43:21 +0000
+Subject: [PATCH 14/70] networks: Test if we can add localhost (IPv6)
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/test-network.c | 7 +++++++
+ 1 file changed, 7 insertions(+)
+
+diff --git a/src/test-network.c b/src/test-network.c
+index 85eca00..8c7e898 100644
+--- a/src/test-network.c
++++ b/src/test-network.c
+@@ -175,6 +175,13 @@ int main(int argc, char** argv) {
+ exit(EXIT_FAILURE);
+ }
+
++ // Try adding localhost
++ err = loc_writer_add_network(writer, &network, "::1/128");
++ if (err != -EINVAL) {
++ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
++ exit(EXIT_FAILURE);
++ }
++
+ FILE* f = tmpfile();
+ if (!f) {
+ fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
+--
+2.20.1
+
+From fc1190aa11e3ff3d2dbf5f4d408c298e7916f46f Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 21 Oct 2020 13:44:50 +0000
+Subject: [PATCH 15/70] networks: Remove accidentially committed debug line
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/network.c | 2 --
+ 1 file changed, 2 deletions(-)
+
+diff --git a/src/network.c b/src/network.c
+index c112a41..be88d75 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -208,8 +208,6 @@ FAIL:
+ if (r)
+ return r;
+
+- DEBUG(ctx, "GOT HERE\n");
+-
+ // Create a new network
+ return loc_network_new(ctx, network, &first_address, prefix);
+ }
+--
+2.20.1
+
+From a1707d8983898b6878cdd5c68744bcc444e278ed Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 21 Oct 2020 13:53:36 +0000
+Subject: [PATCH 16/70] importer: Add search index to announcements table
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index e87d378..d0fe5a6 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -152,6 +152,7 @@ class CLI(object):
+ last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
+ CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
+ CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
++ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
+
+ -- autnums
+ CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
+--
+2.20.1
+
+From 991baf530d47adb2ed7a15b65dc4565d07fa6d07 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 21 Oct 2020 13:54:45 +0000
+Subject: [PATCH 17/70] importer: Add search index to network_overrides table
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index d0fe5a6..fe21d73 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -189,6 +189,8 @@ class CLI(object):
+ );
+ CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
+ ON network_overrides(network);
++ CREATE INDEX IF NOT EXISTS network_overrides_search
++ ON network_overrides USING GIST(network inet_ops);
+ """)
+
+ return db
+--
+2.20.1
+
+From bbea93a74651df10e2ffdbd09eb434dc6a0471bc Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 21 Oct 2020 16:01:57 +0000
+Subject: [PATCH 18/70] importer: Restructure SQL query to be executed in
+ parallel
+
+There are no functional changes, this just runs quicker now.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 67 ++++++++++++++++++---------------
+ 1 file changed, 37 insertions(+), 30 deletions(-)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index fe21d73..3c1e5e2 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -237,34 +237,24 @@ class CLI(object):
+
+ # Select all known networks
+ rows = self.db.query("""
+- -- Get a (sorted) list of all known networks
+- WITH known_networks AS (
+- SELECT network FROM announcements
+- UNION
+- SELECT network FROM networks
+- UNION
+- SELECT network FROM network_overrides
+- ORDER BY network
+- )
+-
+ -- Return a list of those networks enriched with all
+ -- other information that we store in the database
+ SELECT
+- DISTINCT ON (known_networks.network)
+- known_networks.network AS network,
+- announcements.autnum AS autnum,
++ DISTINCT ON (network)
++ network,
++ autnum,
+
+ -- Country
+ COALESCE(
+ (
+ SELECT country FROM network_overrides overrides
+- WHERE announcements.network <<= overrides.network
++ WHERE networks.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT country FROM autnum_overrides overrides
+- WHERE announcements.autnum = overrides.number
++ WHERE networks.autnum = overrides.number
+ ),
+ networks.country
+ ) AS country,
+@@ -273,50 +263,67 @@ class CLI(object):
+ COALESCE(
+ (
+ SELECT is_anonymous_proxy FROM network_overrides overrides
+- WHERE announcements.network <<= overrides.network
++ WHERE networks.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT is_anonymous_proxy FROM autnum_overrides overrides
+- WHERE announcements.autnum = overrides.number
++ WHERE networks.autnum = overrides.number
+ ),
+ FALSE
+ ) AS is_anonymous_proxy,
+ COALESCE(
+ (
+ SELECT is_satellite_provider FROM network_overrides overrides
+- WHERE announcements.network <<= overrides.network
++ WHERE networks.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT is_satellite_provider FROM autnum_overrides overrides
+- WHERE announcements.autnum = overrides.number
++ WHERE networks.autnum = overrides.number
+ ),
+ FALSE
+ ) AS is_satellite_provider,
+ COALESCE(
+ (
+ SELECT is_anycast FROM network_overrides overrides
+- WHERE announcements.network <<= overrides.network
++ WHERE networks.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT is_anycast FROM autnum_overrides overrides
+- WHERE announcements.autnum = overrides.number
++ WHERE networks.autnum = overrides.number
+ ),
+ FALSE
+- ) AS is_anycast,
+-
+- -- Must be part of returned values for ORDER BY clause
+- masklen(announcements.network) AS sort_a,
+- masklen(networks.network) AS sort_b
+- FROM known_networks
+- LEFT JOIN announcements ON known_networks.network <<= announcements.network
+- LEFT JOIN networks ON known_networks.network <<= networks.network
+- ORDER BY known_networks.network, sort_a DESC, sort_b DESC
++ ) AS is_anycast
++ FROM (
++ SELECT
++ known_networks.network AS network,
++ announcements.autnum AS autnum,
++ networks.country AS country,
++
++ -- Must be part of returned values for ORDER BY clause
++ masklen(announcements.network) AS sort_a,
++ masklen(networks.network) AS sort_b
++ FROM (
++ SELECT network FROM announcements
++ UNION ALL
++ SELECT network FROM networks
++ UNION ALL
++ SELECT network FROM network_overrides
++ ) known_networks
++ LEFT JOIN
++ announcements ON known_networks.network <<= announcements.network
++ LEFT JOIN
++ networks ON known_networks.network <<= networks.network
++ ORDER BY
++ known_networks.network,
++ sort_a DESC,
++ sort_b DESC
++ ) networks
+ """)
+
+ for row in rows:
+--
+2.20.1
+
+From 26ab419b68d166f932db1f97c38cb9d793d04187 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 22 Oct 2020 12:24:34 +0000
+Subject: [PATCH 19/70] network: Allow adding single IP addresses and
+ automatically add the prefix
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/network.c | 33 +++++++++++++++------------------
+ src/test-network.c | 4 ++--
+ 2 files changed, 17 insertions(+), 20 deletions(-)
+
+diff --git a/src/network.c b/src/network.c
+index be88d75..d7b1645 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -161,6 +161,7 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
+ const char* address_string) {
+ struct in6_addr first_address;
+ char* prefix_string;
++ unsigned int prefix = 128;
+ int r = -EINVAL;
+
+ DEBUG(ctx, "Attempting to parse network %s\n", address_string);
+@@ -174,21 +175,6 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
+
+ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
+
+- // We need to have a prefix
+- if (!prefix_string) {
+- DEBUG(ctx, "No prefix set\n");
+- goto FAIL;
+- }
+-
+- // Convert prefix to integer
+- unsigned int prefix = strtol(prefix_string, NULL, 10);
+-
+- // End if the prefix was invalid
+- if (!prefix) {
+- DEBUG(ctx, "The prefix is zero or not a number\n");
+- goto FAIL;
+- }
+-
+ // Parse the address
+ r = loc_parse_address(ctx, address_string, &first_address);
+ if (r) {
+@@ -196,9 +182,20 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
+ goto FAIL;
+ }
+
+- // Map the prefix to IPv6 if needed
+- if (IN6_IS_ADDR_V4MAPPED(&first_address))
+- prefix += 96;
++ // If a prefix was given, we will try to parse it
++ if (prefix_string) {
++ // Convert prefix to integer
++ prefix = strtol(prefix_string, NULL, 10);
++
++ if (!prefix) {
++ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
++ goto FAIL;
++ }
++
++ // Map the prefix to IPv6 if needed
++ if (IN6_IS_ADDR_V4MAPPED(&first_address))
++ prefix += 96;
++ }
+
+ FAIL:
+ // Free temporary buffer
+diff --git a/src/test-network.c b/src/test-network.c
+index 8c7e898..b6776b4 100644
+--- a/src/test-network.c
++++ b/src/test-network.c
+@@ -170,8 +170,8 @@ int main(int argc, char** argv) {
+
+ // Try adding a single address
+ err = loc_writer_add_network(writer, &network, "2001:db8::");
+- if (err != -EINVAL) {
+- fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
++ if (err) {
++ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
+ exit(EXIT_FAILURE);
+ }
+
+--
+2.20.1
+
+From aadac4c569e921be1d28dd3b2377ac7f3732213e Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Wed, 21 Oct 2020 14:47:36 +0000
+Subject: [PATCH 20/70] Revert "Revert "Revert "Revert "importer: Import raw
+ sources for inetnum's again""""
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+This reverts commit 44341478233115b26bb27fdb24da5b0a1eedb173.
+
+Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/importer.py | 14 ++++----
+ src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
+ 2 files changed, 70 insertions(+), 7 deletions(-)
+
+diff --git a/src/python/importer.py b/src/python/importer.py
+index de20f37..f19db4b 100644
+--- a/src/python/importer.py
++++ b/src/python/importer.py
+@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
+ "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
+
+ # Asia Pacific Network Information Centre
+- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
++ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
++ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+ #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
+ #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
+ "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
+@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
+ # XXX ???
+
+ # Réseaux IP Européens
+- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
+@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
+
+ EXTENDED_SOURCES = (
+ # African Network Information Centre
+- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
++ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+
+ # Asia Pacific Network Information Centre
+- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
++ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+
+ # American Registry for Internet Numbers
+ "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
+@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
+ "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
+
+ # Réseaux IP Européens
+- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
++ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+ )
+
+ class Downloader(object):
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index 3c1e5e2..e8a4fc5 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -405,6 +405,10 @@ class CLI(object):
+ if line.startswith("aut-num:"):
+ return self._parse_autnum_block(block)
+
++ # inetnum
++ if line.startswith("inet6num:") or line.startswith("inetnum:"):
++ return self._parse_inetnum_block(block)
++
+ # organisation
+ elif line.startswith("organisation:"):
+ return self._parse_org_block(block)
+@@ -434,6 +438,65 @@ class CLI(object):
+ autnum.get("asn"), autnum.get("org"),
+ )
+
++ def _parse_inetnum_block(self, block):
++ logging.debug("Parsing inetnum block:")
++
++ inetnum = {}
++ for line in block:
++ logging.debug(line)
++
++ # Split line
++ key, val = split_line(line)
++
++ if key == "inetnum":
++ start_address, delim, end_address = val.partition("-")
++
++ # Strip any excess space
++ start_address, end_address = start_address.rstrip(), end_address.strip()
++
++ # Convert to IP address
++ try:
++ start_address = ipaddress.ip_address(start_address)
++ end_address = ipaddress.ip_address(end_address)
++ except ValueError:
++ logging.warning("Could not parse line: %s" % line)
++ return
++
++ # Set prefix to default
++ prefix = 32
++
++ # Count number of addresses in this subnet
++ num_addresses = int(end_address) - int(start_address)
++ if num_addresses:
++ prefix -= math.log(num_addresses, 2)
++
++ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
++
++ elif key == "inet6num":
++ inetnum[key] = val
++
++ elif key == "country":
++ if val == "UNITED STATES":
++ val = "US"
++
++ inetnum[key] = val.upper()
++
++ # Skip empty objects
++ if not inetnum:
++ return
++
++ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
++
++ # Bail out in case we have processed a non-public IP network
++ if network.is_private:
++ logging.warning("Skipping non-globally routable network: %s" % network)
++ return
++
++ self.db.execute("INSERT INTO networks(network, country) \
++ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
++ "%s" % network, inetnum.get("country"),
++ )
++
+ def _parse_org_block(self, block):
+ org = {}
+ for line in block:
+--
+2.20.1
+
+From 002deb6b42ac0b3624c07e3352cebd72dc0685a2 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Wed, 21 Oct 2020 14:47:37 +0000
+Subject: [PATCH 21/70] Revert "Revert "location-importer.in: only import
+ relevant data from AFRINIC, APNIC and RIPE""
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+This reverts commit 13f67f285856e8eabfeff2daf1be3aeaa36a82cc.
+
+Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
+ 1 file changed, 87 insertions(+), 2 deletions(-)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index e8a4fc5..5656c41 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -166,6 +166,7 @@ class CLI(object):
+ -- networks
+ CREATE TABLE IF NOT EXISTS networks(network inet, country text);
+ CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
++ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
+ CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
+
+ -- overrides
+@@ -375,6 +376,16 @@ class CLI(object):
+ CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
+ ON COMMIT DROP;
+ CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
++
++ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
++ ON COMMIT DROP;
++ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
++ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
++ """)
++
++ # Remove all previously imported content
++ self.db.execute("""
++ TRUNCATE TABLE networks;
+ """)
+
+ for source in location.importer.WHOIS_SOURCES:
+@@ -382,6 +393,67 @@ class CLI(object):
+ for block in f:
+ self._parse_block(block)
+
++ # Process all parsed networks from every RIR we happen to have access to,
++ # insert the largest network chunks into the networks table immediately...
++ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
++
++ for family in (row.family for row in families):
++ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
++
++ self.db.execute("INSERT INTO networks(network, country) \
++ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
++
++ # ... determine any other prefixes for this network family, ...
++ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
++ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
++
++ # ... and insert networks with this prefix in case they provide additional
++ # information (i. e. subnet of a larger chunk with a different country)
++ for prefix in (row.prefix for row in prefixes):
++ self.db.execute("""
++ WITH candidates AS (
++ SELECT
++ _rirdata.network,
++ _rirdata.country
++ FROM
++ _rirdata
++ WHERE
++ family(_rirdata.network) = %s
++ AND
++ masklen(_rirdata.network) = %s
++ ),
++ filtered AS (
++ SELECT
++ DISTINCT ON (c.network)
++ c.network,
++ c.country,
++ masklen(networks.network),
++ networks.country AS parent_country
++ FROM
++ candidates c
++ LEFT JOIN
++ networks
++ ON
++ c.network << networks.network
++ ORDER BY
++ c.network,
++ masklen(networks.network) DESC NULLS LAST
++ )
++ INSERT INTO
++ networks(network, country)
++ SELECT
++ network,
++ country
++ FROM
++ filtered
++ WHERE
++ parent_country IS NULL
++ OR
++ country <> parent_country
++ ON CONFLICT DO NOTHING""",
++ family, prefix,
++ )
++
+ self.db.execute("""
+ INSERT INTO autnums(number, name)
+ SELECT _autnums.number, _organizations.name FROM _autnums
+@@ -482,17 +554,30 @@ class CLI(object):
+ inetnum[key] = val.upper()
+
+ # Skip empty objects
+- if not inetnum:
++ if not inetnum or not "country" in inetnum:
+ return
+
+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
++ # Bail out in case we have processed a network covering the entire IP range, which
++ # is necessary to work around faulty (?) IPv6 network processing
++ if network.prefixlen == 0:
++ logging.warning("Skipping network covering the entire IP adress range: %s" % network)
++ return
++
++ # Bail out in case we have processed a network whose prefix length indicates it is
++ # not globally routable (we have decided not to process them at the moment, as they
++ # significantly enlarge our database without providing very helpful additional information)
++ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
++ logging.info("Skipping network too small to be publicly announced: %s" % network)
++ return
++
+ # Bail out in case we have processed a non-public IP network
+ if network.is_private:
+ logging.warning("Skipping non-globally routable network: %s" % network)
+ return
+
+- self.db.execute("INSERT INTO networks(network, country) \
++ self.db.execute("INSERT INTO _rirdata(network, country) \
+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+ "%s" % network, inetnum.get("country"),
+ )
+--
+2.20.1
+
+From 28c73fa3f4257e0a41e52af8a9643da414a6cb6f Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Wed, 21 Oct 2020 14:47:38 +0000
+Subject: [PATCH 22/70] export.py: fix exporting IP networks for crappy
+ xt_geoip module
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+In contrast to the location database itself, the xt_geoip module
+consumes a list of IP networks for each country, and returns after the
+first match.
+
+We therefore need to...
+
+(a) sort IP networks by their size, allow as precise matches as possible
+(b) export _any_ IP networks - including inverted subnets - to prevent
+ undefined overlaps
+(c) do the entire thing as fast as possible, consuming as less disk
+ space as possible, which is why we can't just iterate over all /24
+ chunks
+
+Partially fixes: #12499
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/export.py | 69 ++++++++++++++++++++++++++++++++++----------
+ 1 file changed, 54 insertions(+), 15 deletions(-)
+
+diff --git a/src/python/export.py b/src/python/export.py
+index d15c6f0..5eaf43f 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -39,8 +39,8 @@ class OutputWriter(object):
+ suffix = "networks"
+ mode = "w"
+
+- def __init__(self, f, prefix=None, flatten=True):
+- self.f, self.prefix, self.flatten = f, prefix, flatten
++ def __init__(self, db, f, prefix=None, flatten=True):
++ self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten
+
+ # The previously written network
+ self._last_network = None
+@@ -49,13 +49,13 @@ class OutputWriter(object):
+ self._write_header()
+
+ @classmethod
+- def open(cls, filename, **kwargs):
++ def open(cls, db, filename, **kwargs):
+ """
+ Convenience function to open a file
+ """
+ f = open(filename, cls.mode)
+
+- return cls(f, **kwargs)
++ return cls(db, f, **kwargs)
+
+ def __repr__(self):
+ return "<%s f=%s>" % (self.__class__.__name__, self.f)
+@@ -87,13 +87,31 @@ class OutputWriter(object):
+ def _write_network(self, network):
+ self.f.write("%s\n" % network)
+
+- def write(self, network):
++ def write(self, network, subnets):
+ if self.flatten and self._flatten(network):
+ log.debug("Skipping writing network %s" % network)
+ return
+
+- # Write the network to file
+- self._write_network(network)
++ # Write the network when it has no subnets
++ if not subnets:
++ network = ipaddress.ip_network("%s" % network)
++ return self._write_network(network)
++
++ # Collect all matching subnets
++ matching_subnets = []
++
++ for subnet in sorted(subnets):
++ # Try to find the subnet in the database
++ n = self.db.lookup("%s" % subnet.network_address)
++
++ # No entry or matching country means those IP addresses belong here
++ if not n or n.country_code == network.country_code:
++ matching_subnets.append(subnet)
++
++ # Write all networks as compact as possible
++ for network in ipaddress.collapse_addresses(matching_subnets):
++ log.debug("Writing %s to database" % network)
++ self._write_network(network)
+
+ def finish(self):
+ """
+@@ -143,10 +161,10 @@ class XTGeoIPOutputWriter(OutputWriter):
+ mode = "wb"
+
+ def _write_network(self, network):
+- for address in (network.first_address, network.last_address):
++ for address in (network.network_address, network.broadcast_address):
+ # Convert this into a string of bits
+ bytes = socket.inet_pton(
+- network.family, address,
++ socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
+ )
+
+ self.f.write(bytes)
+@@ -175,7 +193,7 @@ class Exporter(object):
+ directory, prefix=country_code, suffix=self.writer.suffix, family=family,
+ )
+
+- writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
++ writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code)
+
+ # Create writers for ASNs
+ for asn in asns:
+@@ -183,22 +201,43 @@ class Exporter(object):
+ directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
+ )
+
+- writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
++ writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
+
+ # Get all networks that match the family
+ networks = self.db.search_networks(family=family)
+
++ # Materialise the generator into a list (uses quite some memory)
++ networks = list(networks)
++
+ # Walk through all networks
+- for network in networks:
++ for i, network in enumerate(networks):
++ _network = ipaddress.ip_network("%s" % network)
++
++ # Search for all subnets
++ subnets = set()
++
++ while i < len(networks):
++ subnet = networks[i+1]
++
++ if subnet.is_subnet_of(network):
++ _subnet = ipaddress.ip_network("%s" % subnet)
++
++ subnets.add(_subnet)
++ subnets.update(_network.address_exclude(_subnet))
++
++ i += 1
++ else:
++ break
++
+ # Write matching countries
+ try:
+- writers[network.country_code].write(network)
++ writers[network.country_code].write(network, subnets)
+ except KeyError:
+ pass
+
+ # Write matching ASNs
+ try:
+- writers[network.asn].write(network)
++ writers[network.asn].write(network, subnets)
+ except KeyError:
+ pass
+
+@@ -209,7 +248,7 @@ class Exporter(object):
+ country = flags[flag]
+
+ try:
+- writers[country].write(network)
++ writers[country].write(network, subnets)
+ except KeyError:
+ pass
+
+--
+2.20.1
+
+From bd341642fc6bbcc050e9b4ec5124585c83cab84d Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Wed, 21 Oct 2020 14:47:39 +0000
+Subject: [PATCH 23/70] location-importer.in: filter bogus IP networks for both
+ Whois and extended sources
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Sanity checks for parsed networks have been put into a separate function
+to avoid boilerplate code for extended sources. This makes the location
+database less vulnerable to garbage written into RIR databases on
+purpose or by chance.
+
+Fixes: #12500
+
+Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 83 ++++++++++++++++++++++++++-------
+ 1 file changed, 67 insertions(+), 16 deletions(-)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index 5656c41..f24d357 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -469,6 +469,69 @@ class CLI(object):
+ for line in f:
+ self._parse_line(line)
+
++ def _check_parsed_network(self, network):
++ """
++ Assistive function to detect and subsequently sort out parsed
++ networks from RIR data (both Whois and so-called "extended sources"),
++ which are or have...
++
++ (a) not globally routable (RFC 1918 space, et al.)
++ (b) covering a too large chunk of the IP address space (prefix length
++ is < 7 for IPv4 networks, and < 10 for IPv6)
++ (c) "0.0.0.0" or "::" as a network address
++ (d) are too small for being publicly announced (we have decided not to
++ process them at the moment, as they significantly enlarge our
++ database without providing very helpful additional information)
++
++ This unfortunately is necessary due to brain-dead clutter across
++ various RIR databases, causing mismatches and eventually disruptions.
++
++ We will return False in case a network is not suitable for adding
++ it to our database, and True otherwise.
++ """
++
++ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
++ return False
++
++ if not network.is_global:
++ logging.warning("Skipping non-globally routable network: %s" % network)
++ return False
++
++ if network.version == 4:
++ if network.prefixlen < 7:
++ logging.warning("Skipping too big IP chunk: %s" % network)
++ return False
++
++ if network.prefixlen > 24:
++ logging.info("Skipping network too small to be publicly announced: %s" % network)
++ return False
++
++ if str(network.network_address) == "0.0.0.0":
++ logging.warning("Skipping network based on 0.0.0.0: %s" % network)
++ return False
++
++ elif network.version == 6:
++ if network.prefixlen < 10:
++ logging.warning("Skipping too big IP chunk: %s" % network)
++ return False
++
++ if network.prefixlen > 48:
++ logging.info("Skipping network too small to be publicly announced: %s" % network)
++ return False
++
++ if str(network.network_address) == "::":
++ logging.warning("Skipping network based on '::': %s" % network)
++ return False
++
++ else:
++ # This should not happen...
++ logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
++ return False
++
++ # In case we have made it here, the network is considered to
++ # be suitable for libloc consumption...
++ return True
++
+ def _parse_block(self, block):
+ # Get first line to find out what type of block this is
+ line = block[0]
+@@ -559,22 +622,7 @@ class CLI(object):
+
+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
+- # Bail out in case we have processed a network covering the entire IP range, which
+- # is necessary to work around faulty (?) IPv6 network processing
+- if network.prefixlen == 0:
+- logging.warning("Skipping network covering the entire IP adress range: %s" % network)
+- return
+-
+- # Bail out in case we have processed a network whose prefix length indicates it is
+- # not globally routable (we have decided not to process them at the moment, as they
+- # significantly enlarge our database without providing very helpful additional information)
+- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
+- logging.info("Skipping network too small to be publicly announced: %s" % network)
+- return
+-
+- # Bail out in case we have processed a non-public IP network
+- if network.is_private:
+- logging.warning("Skipping non-globally routable network: %s" % network)
++ if not self._check_parsed_network(network):
+ return
+
+ self.db.execute("INSERT INTO _rirdata(network, country) \
+@@ -658,6 +706,9 @@ class CLI(object):
+ log.warning("Invalid IP address: %s" % address)
+ return
+
++ if not self._check_parsed_network(network):
++ return
++
+ self.db.execute("INSERT INTO networks(network, country) \
+ VALUES(%s, %s) ON CONFLICT (network) DO \
+ UPDATE SET country = excluded.country",
+--
+2.20.1
+
+From eee65490a10e0fe89b3834b8be176fc900084fa0 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Wed, 21 Oct 2020 14:47:40 +0000
+Subject: [PATCH 24/70] importer.py: fetch LACNIC data via HTTPS
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/importer.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/python/importer.py b/src/python/importer.py
+index f19db4b..5f46bc3 100644
+--- a/src/python/importer.py
++++ b/src/python/importer.py
+@@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
+ "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
+
+ # Latin America and Caribbean Network Information Centre
+- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
++ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
+
+ # Réseaux IP Européens
+ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+--
+2.20.1
+
+From 84187ab5436eb158529d6f5e2a38890b4af3ddb4 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Wed, 21 Oct 2020 14:47:41 +0000
+Subject: [PATCH 25/70] location-importer.in: omit historic/orphaned RIR data
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Some RIRs include detailled information regarding networks not managed
+by or allocated to themselves, particually APNIC. We need to filter
+those networks (they usually have a characteristic network name) in
+order to prevent operational quirks or returning wrong country codes.
+
+Fixes: #12501
+Partially fixes: #12499
+
+Cc: Michael Tremer <michael.tremer@ipfire.org>
+Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 38 +++++++++++++++++++++------------
+ 1 file changed, 24 insertions(+), 14 deletions(-)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index f24d357..a869256 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -494,38 +494,38 @@ class CLI(object):
+ return False
+
+ if not network.is_global:
+- logging.warning("Skipping non-globally routable network: %s" % network)
++ log.warning("Skipping non-globally routable network: %s" % network)
+ return False
+
+ if network.version == 4:
+ if network.prefixlen < 7:
+- logging.warning("Skipping too big IP chunk: %s" % network)
++ log.warning("Skipping too big IP chunk: %s" % network)
+ return False
+
+ if network.prefixlen > 24:
+- logging.info("Skipping network too small to be publicly announced: %s" % network)
++ log.info("Skipping network too small to be publicly announced: %s" % network)
+ return False
+
+ if str(network.network_address) == "0.0.0.0":
+- logging.warning("Skipping network based on 0.0.0.0: %s" % network)
++ log.warning("Skipping network based on 0.0.0.0: %s" % network)
+ return False
+
+ elif network.version == 6:
+ if network.prefixlen < 10:
+- logging.warning("Skipping too big IP chunk: %s" % network)
++ log.warning("Skipping too big IP chunk: %s" % network)
+ return False
+
+ if network.prefixlen > 48:
+- logging.info("Skipping network too small to be publicly announced: %s" % network)
++ log.info("Skipping network too small to be publicly announced: %s" % network)
+ return False
+
+ if str(network.network_address) == "::":
+- logging.warning("Skipping network based on '::': %s" % network)
++ log.warning("Skipping network based on '::': %s" % network)
+ return False
+
+ else:
+ # This should not happen...
+- logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
++ log.warning("Skipping network of unknown family, this should not happen: %s" % network)
+ return False
+
+ # In case we have made it here, the network is considered to
+@@ -574,15 +574,22 @@ class CLI(object):
+ )
+
+ def _parse_inetnum_block(self, block):
+- logging.debug("Parsing inetnum block:")
++ log.debug("Parsing inetnum block:")
+
+ inetnum = {}
+ for line in block:
+- logging.debug(line)
++ log.debug(line)
+
+ # Split line
+ key, val = split_line(line)
+
++ # Filter any inetnum records which are only referring to IP space
++ # not managed by that specific RIR...
++ if key == "netname":
++ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
++ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
++ return
++
+ if key == "inetnum":
+ start_address, delim, end_address = val.partition("-")
+
+@@ -594,7 +601,7 @@ class CLI(object):
+ start_address = ipaddress.ip_address(start_address)
+ end_address = ipaddress.ip_address(end_address)
+ except ValueError:
+- logging.warning("Could not parse line: %s" % line)
++ log.warning("Could not parse line: %s" % line)
+ return
+
+ # Set prefix to default
+@@ -611,15 +618,18 @@ class CLI(object):
+ inetnum[key] = val
+
+ elif key == "country":
+- if val == "UNITED STATES":
+- val = "US"
+-
+ inetnum[key] = val.upper()
+
+ # Skip empty objects
+ if not inetnum or not "country" in inetnum:
+ return
+
++ # Skip objects with bogus country code 'ZZ'
++ if inetnum.get("country") == "ZZ":
++ log.warning("Skipping network with bogus country 'ZZ': %s" % \
++ (inetnum.get("inet6num") or inetnum.get("inetnum")))
++ return
++
+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
+ if not self._check_parsed_network(network):
+--
+2.20.1
+
+From ebb087cfa30ec5ca0c96dcce66a91245c1ffc271 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Wed, 21 Oct 2020 14:47:43 +0000
+Subject: [PATCH 26/70] location-importer.in: avoid log spam for too small
+ networks
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index a869256..864eab1 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -503,7 +503,7 @@ class CLI(object):
+ return False
+
+ if network.prefixlen > 24:
+- log.info("Skipping network too small to be publicly announced: %s" % network)
++ log.debug("Skipping network too small to be publicly announced: %s" % network)
+ return False
+
+ if str(network.network_address) == "0.0.0.0":
+@@ -516,7 +516,7 @@ class CLI(object):
+ return False
+
+ if network.prefixlen > 48:
+- log.info("Skipping network too small to be publicly announced: %s" % network)
++ log.debug("Skipping network too small to be publicly announced: %s" % network)
+ return False
+
+ if str(network.network_address) == "::":
+--
+2.20.1
+
+From bbed1fd2330e8efa6b413dc152a1a6ef2d771aac Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 27 Oct 2020 17:14:30 +0000
+Subject: [PATCH 27/70] export: Flatten the tree before exporting it
+
+This patch removes the possibility that any IP address ranges
+might show up in multiple exported files.
+
+If this was content from the database:
+
+ * 10.0.0.0/16 - DE
+ * 10.0.1.0/24 - FR
+
+Then the IP address 10.0.1.1 would match for both countries.
+
+The algorithm will now break the larger /16 subnet apart into
+smaller subnets so that 10.0.1.0/24 is no longer overlapped.
+
+There was some time spent on this to make this as efficient
+as possible.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/export.py | 154 ++++++++++++++++++++++++++++++-------------
+ 1 file changed, 110 insertions(+), 44 deletions(-)
+
+diff --git a/src/python/export.py b/src/python/export.py
+index 5eaf43f..dd44332 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -89,28 +89,55 @@ class OutputWriter(object):
+
+ def write(self, network, subnets):
+ if self.flatten and self._flatten(network):
+- log.debug("Skipping writing network %s" % network)
++ log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
+ return
+
++ # Convert network into a Python object
++ _network = ipaddress.ip_network("%s" % network)
++
+ # Write the network when it has no subnets
+ if not subnets:
+- network = ipaddress.ip_network("%s" % network)
+- return self._write_network(network)
++ log.debug("Writing %s to %s" % (_network, self.f))
++ return self._write_network(_network)
++
++ # Convert subnets into Python objects
++ _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
++
++ # Split the network into smaller bits so that
++ # we can accomodate for any gaps in it later
++ to_check = set()
++ for _subnet in _subnets:
++ to_check.update(
++ _network.address_exclude(_subnet)
++ )
++
++ # Clear the list of all subnets
++ subnets = []
++
++ # Check if all subnets to not overlap with anything else
++ while to_check:
++ subnet_to_check = to_check.pop()
+
+- # Collect all matching subnets
+- matching_subnets = []
++ for _subnet in _subnets:
++ # Drop this subnet if it equals one of the subnets
++ # or if it is subnet of one of them
++ if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
++ break
+
+- for subnet in sorted(subnets):
+- # Try to find the subnet in the database
+- n = self.db.lookup("%s" % subnet.network_address)
++ # Break it down if it overlaps
++ if subnet_to_check.overlaps(_subnet):
++ to_check.update(
++ subnet_to_check.address_exclude(_subnet)
++ )
++ break
+
+- # No entry or matching country means those IP addresses belong here
+- if not n or n.country_code == network.country_code:
+- matching_subnets.append(subnet)
++ # Add the subnet again as it passed the check
++ else:
++ subnets.append(subnet_to_check)
+
+ # Write all networks as compact as possible
+- for network in ipaddress.collapse_addresses(matching_subnets):
+- log.debug("Writing %s to database" % network)
++ for network in ipaddress.collapse_addresses(subnets):
++ log.debug("Writing %s to %s" % (network, self.f))
+ self._write_network(network)
+
+ def finish(self):
+@@ -206,40 +233,40 @@ class Exporter(object):
+ # Get all networks that match the family
+ networks = self.db.search_networks(family=family)
+
+- # Materialise the generator into a list (uses quite some memory)
+- networks = list(networks)
++ # Create a stack with all networks in order where we can put items back
++ # again and retrieve them in the next iteration.
++ networks = BufferedStack(networks)
+
+ # Walk through all networks
+- for i, network in enumerate(networks):
+- _network = ipaddress.ip_network("%s" % network)
+-
+- # Search for all subnets
+- subnets = set()
+-
+- while i < len(networks):
+- subnet = networks[i+1]
+-
+- if subnet.is_subnet_of(network):
+- _subnet = ipaddress.ip_network("%s" % subnet)
+-
+- subnets.add(_subnet)
+- subnets.update(_network.address_exclude(_subnet))
+-
+- i += 1
+- else:
++ for network in networks:
++ # Collect all networks which are a subnet of network
++ subnets = []
++ for subnet in networks:
++ # If the next subnet was not a subnet, we have to push
++ # it back on the stack and break this loop
++ if not subnet.is_subnet_of(network):
++ networks.push(subnet)
+ break
+
++ subnets.append(subnet)
++
+ # Write matching countries
+- try:
+- writers[network.country_code].write(network, subnets)
+- except KeyError:
+- pass
++ if network.country_code and network.country_code in writers:
++ # Mismatching subnets
++ gaps = [
++ subnet for subnet in subnets if not network.country_code == subnet.country_code
++ ]
++
++ writers[network.country_code].write(network, gaps)
+
+ # Write matching ASNs
+- try:
+- writers[network.asn].write(network, subnets)
+- except KeyError:
+- pass
++ if network.asn and network.asn in writers:
++ # Mismatching subnets
++ gaps = [
++ subnet for subnet in subnets if not network.asn == subnet.asn
++ ]
++
++ writers[network.asn].write(network, gaps)
+
+ # Handle flags
+ for flag in flags:
+@@ -247,10 +274,19 @@ class Exporter(object):
+ # Fetch the "fake" country code
+ country = flags[flag]
+
+- try:
+- writers[country].write(network, subnets)
+- except KeyError:
+- pass
++ if not country in writers:
++ continue
++
++ gaps = [
++ subnet for subnet in subnets
++ if not subnet.has_flag(flag)
++ ]
++
++ writers[country].write(network, gaps)
++
++ # Push all subnets back onto the stack
++ for subnet in reversed(subnets):
++ networks.push(subnet)
+
+ # Write everything to the filesystem
+ for writer in writers.values():
+@@ -262,3 +298,33 @@ class Exporter(object):
+ )
+
+ return os.path.join(directory, filename)
++
++
++class BufferedStack(object):
++ """
++ This class takes an iterator and when being iterated
++ over it returns objects from that iterator for as long
++ as there are any.
++
++ It additionally has a function to put an item back on
++ the back so that it will be returned again at the next
++ iteration.
++ """
++ def __init__(self, iterator):
++ self.iterator = iterator
++ self.stack = []
++
++ def __iter__(self):
++ return self
++
++ def __next__(self):
++ if self.stack:
++ return self.stack.pop(0)
++
++ return next(self.iterator)
++
++ def push(self, elem):
++ """
++ Takes an element and puts it on the stack
++ """
++ self.stack.insert(0, elem)
+--
+2.20.1
+
+From e99a72265c1ba2194b61663eda7e9f14e0083016 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 28 Oct 2020 09:52:36 +0000
+Subject: [PATCH 28/70] location: Fix Python syntax error in verify()
+
+The database is now being opened before the requested
+method is called and handle_verify() wasn't updated.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location.in | 8 +-------
+ 1 file changed, 1 insertion(+), 7 deletions(-)
+
+diff --git a/src/python/location.in b/src/python/location.in
+index 44ad726..b5e5758 100644
+--- a/src/python/location.in
++++ b/src/python/location.in
+@@ -453,13 +453,7 @@ class CLI(object):
+
+ return 0
+
+- def handle_verify(self, ns):
+- try:
+- db = location.Database(ns.database)
+- except FileNotFoundError as e:
+- log.error("%s: %s" % (ns.database, e))
+- return 127
+-
++ def handle_verify(self, db, ns):
+ # Verify the database
+ with open(ns.public_key, "r") as f:
+ if not db.verify(f):
+--
+2.20.1
+
+From 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Thu, 29 Oct 2020 07:25:53 -0700
+Subject: [PATCH 29/70] location update: Remove double conversion of timestamps
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+This caused that the timestamp in DNS was misinterpreted
+as local time and often, databases could not be downloaded.
+
+Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
+---
+ src/python/location.in | 5 +----
+ 1 file changed, 1 insertion(+), 4 deletions(-)
+
+diff --git a/src/python/location.in b/src/python/location.in
+index b5e5758..070640c 100644
+--- a/src/python/location.in
++++ b/src/python/location.in
+@@ -421,11 +421,8 @@ class CLI(object):
+ # Fetch the timestamp we need from DNS
+ t = location.discover_latest_version()
+
+- # Parse timestamp into datetime format
+- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
+-
+ # Check the version of the local database
+- if db and timestamp and db.created_at >= timestamp.timestamp():
++ if db and t and db.created_at >= t:
+ log.info("Already on the latest version")
+ return
+
+--
+2.20.1
+
+From 60c1ac0307312614bd6980d30b44bb59b5a6ab6e Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Thu, 29 Oct 2020 07:36:46 -0700
+Subject: [PATCH 30/70] location.in: do not confuse UTC with local time zones
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
+---
+ src/python/location.in | 12 +++++-------
+ 1 file changed, 5 insertions(+), 7 deletions(-)
+
+diff --git a/src/python/location.in b/src/python/location.in
+index 070640c..0d09210 100644
+--- a/src/python/location.in
++++ b/src/python/location.in
+@@ -398,10 +398,7 @@ class CLI(object):
+
+ def handle_update(self, db, ns):
+ if ns.cron and db:
+- now = datetime.datetime.utcnow()
+-
+- # Parse the database timestamp
+- t = datetime.datetime.utcfromtimestamp(db.created_at)
++ now = time.time()
+
+ if ns.cron == "daily":
+ delta = datetime.timedelta(days=1)
+@@ -410,11 +407,12 @@ class CLI(object):
+ elif ns.cron == "monthly":
+ delta = datetime.timedelta(days=30)
+
++ delta = delta.total_seconds()
++
+ # Check if the database has recently been updated
+- if t >= (now - delta):
++ if db.created_at >= (now - delta):
+ log.info(
+- _("The database has been updated recently (%s)") % \
+- format_timedelta(now - t),
++ _("The database has been updated recently"),
+ )
+ return 3
+
+--
+2.20.1
+
+From e7d612e5219ef9ba612ed404e4e2c174110d3dd7 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
+Date: Tue, 3 Nov 2020 15:31:08 +0000
+Subject: [PATCH 31/70] location-importer.in: always convert organisation
+ handles into upper cases
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Fixes: #12523
+
+Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location-importer.in | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index 864eab1..2dec89e 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -560,7 +560,7 @@ class CLI(object):
+ autnum["asn"] = m.group(2)
+
+ elif key == "org":
+- autnum[key] = val
++ autnum[key] = val.upper()
+
+ # Skip empty objects
+ if not autnum:
+@@ -646,7 +646,9 @@ class CLI(object):
+ # Split line
+ key, val = split_line(line)
+
+- if key in ("organisation", "org-name"):
++ if key == "organisation":
++ org[key] = val.upper()
++ elif key == "org-name":
+ org[key] = val
+
+ # Skip empty objects
+--
+2.20.1
+
+From e96704f43acca1a8f56d9a680cce281f5e587ec5 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 11 Nov 2020 21:16:45 +0000
+Subject: [PATCH 32/70] test: Add tests for database enumerator
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/test-database.c | 53 +++++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 53 insertions(+)
+
+diff --git a/src/test-database.c b/src/test-database.c
+index b4a75c4..4aef94e 100644
+--- a/src/test-database.c
++++ b/src/test-database.c
+@@ -38,6 +38,14 @@ const char* DESCRIPTION =
+ "Maecenas ut venenatis nunc.";
+ const char* LICENSE = "CC";
+
++const char* networks[] = {
++ "2001:db8::/32",
++ "2001:db8:1000::/48",
++ "2001:db8:2000::/48",
++ "2001:db8:2020::/48",
++ NULL,
++};
++
+ static int attempt_to_open(struct loc_ctx* ctx, char* path) {
+ FILE* f = fopen(path, "r");
+ if (!f)
+@@ -139,6 +147,24 @@ int main(int argc, char** argv) {
+ exit(EXIT_FAILURE);
+ }
+
++ struct loc_network* network = NULL;
++
++ // Add some networks
++ const char** n = networks;
++ while (*n) {
++ err = loc_writer_add_network(writer, &network, *n);
++ if (err) {
++ fprintf(stderr, "Could not add network %s\n", *n);
++ exit(EXIT_FAILURE);
++ }
++
++ // Set a country
++ loc_network_set_country_code(network, "XX");
++
++ // Next one
++ n++;
++ }
++
+ FILE* f = tmpfile();
+ if (!f) {
+ fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
+@@ -170,6 +196,33 @@ int main(int argc, char** argv) {
+ exit(EXIT_FAILURE);
+ }
+
++ // Enumerator
++ struct loc_database_enumerator* enumerator;
++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
++ if (err) {
++ fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
++ exit(EXIT_FAILURE);
++ }
++
++ // Walk through all networks
++ while (1) {
++ err = loc_database_enumerator_next_network(enumerator, &network);
++ if (err) {
++ fprintf(stderr, "Error fetching the next network: %d\n", err);
++ exit(EXIT_FAILURE);
++ }
++
++ if (!network)
++ break;
++
++ char* s = loc_network_str(network);
++ printf("Got network: %s\n", s);
++ free(s);
++ }
++
++ // Free the enumerator
++ loc_database_enumerator_unref(enumerator);
++
+ // Close the database
+ loc_database_unref(db);
+ loc_unref(ctx);
+--
+2.20.1
+
+From ecce288da39a2c0eb60076050ca21e9619f61844 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 11 Nov 2020 23:01:19 +0000
+Subject: [PATCH 33/70] networks: Add list to manage groups of networks
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/libloc.sym | 11 ++++++
+ src/loc/network.h | 12 ++++++
+ src/network.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++
+ 3 files changed, 119 insertions(+)
+
+diff --git a/src/libloc.sym b/src/libloc.sym
+index b8296eb..6ef2d27 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -98,6 +98,17 @@ global:
+ loc_network_str;
+ loc_network_unref;
+
++ # Network List
++ loc_network_list_clear;
++ loc_network_list_empty;
++ loc_network_list_get;
++ loc_network_list_new;
++ loc_network_list_pop;
++ loc_network_list_push;
++ loc_network_list_ref;
++ loc_network_list_size;
++ loc_network_list_unref;
++
+ # Writer
+ loc_writer_add_as;
+ loc_writer_add_country;
+diff --git a/src/loc/network.h b/src/loc/network.h
+index 70c3803..fd20812 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -56,6 +56,18 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
+
+ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
+
++// List
++struct loc_network_list;
++int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
++struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
++struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
++size_t loc_network_list_size(struct loc_network_list* list);
++int loc_network_list_empty(struct loc_network_list* list);
++void loc_network_list_clear(struct loc_network_list* list);
++struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
++int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
++struct loc_network* loc_network_list_pop(struct loc_network_list* list);
++
+ #ifdef LIBLOC_PRIVATE
+
+ int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
+diff --git a/src/network.c b/src/network.c
+index d7b1645..c9e7979 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -746,3 +746,99 @@ LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node)
+ LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
+ return loc_network_ref(node->network);
+ }
++
++// List
++
++struct loc_network_list {
++ struct loc_ctx* ctx;
++ int refcount;
++
++ struct loc_network* list[1024];
++ size_t size;
++ size_t max_size;
++};
++
++LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
++ struct loc_network_list** list) {
++ struct loc_network_list* l = calloc(1, sizeof(*l));
++ if (!l)
++ return -ENOMEM;
++
++ l->ctx = loc_ref(ctx);
++ l->refcount = 1;
++
++ // Do not allow this list to grow larger than this
++ l->max_size = 1024;
++
++ DEBUG(l->ctx, "Network list allocated at %p\n", l);
++ *list = l;
++ return 0;
++}
++
++LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
++ list->refcount++;
++
++ return list;
++}
++
++static void loc_network_list_free(struct loc_network_list* list) {
++ DEBUG(list->ctx, "Releasing network list at %p\n", list);
++
++ for (unsigned int i = 0; i < list->size; i++)
++ loc_network_unref(list->list[i]);
++
++ loc_unref(list->ctx);
++ free(list);
++}
++
++LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
++ if (!list)
++ return NULL;
++
++ if (--list->refcount > 0)
++ return list;
++
++ loc_network_list_free(list);
++ return NULL;
++}
++
++LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
++ return list->size;
++}
++
++LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
++ return list->size == 0;
++}
++
++LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
++ for (unsigned int i = 0; i < list->size; i++)
++ loc_network_unref(list->list[i]);
++
++ list->size = 0;
++}
++
++LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
++ // Check index
++ if (index >= list->size)
++ return NULL;
++
++ return loc_network_ref(list->list[index]);
++}
++
++LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
++ // Check if we have space left
++ if (list->size == list->max_size)
++ return -ENOMEM;
++
++ list->list[list->size++] = loc_network_ref(network);
++
++ return 0;
++}
++
++LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
++ // Return nothing when empty
++ if (loc_network_list_empty(list))
++ return NULL;
++
++ return list->list[list->size--];
++}
+--
+2.20.1
+
+From 8b2205272b7872a1458ad87811abf58609f38ad4 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 13:57:35 +0000
+Subject: [PATCH 34/70] networks: Add function to dump lists
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/libloc.sym | 1 +
+ src/loc/network.h | 1 +
+ src/network.c | 14 ++++++++++++++
+ 3 files changed, 16 insertions(+)
+
+diff --git a/src/libloc.sym b/src/libloc.sym
+index 6ef2d27..a5641c6 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -100,6 +100,7 @@ global:
+
+ # Network List
+ loc_network_list_clear;
++ loc_network_list_dump;
+ loc_network_list_empty;
+ loc_network_list_get;
+ loc_network_list_new;
+diff --git a/src/loc/network.h b/src/loc/network.h
+index fd20812..44c50a4 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -64,6 +64,7 @@ struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
+ size_t loc_network_list_size(struct loc_network_list* list);
+ int loc_network_list_empty(struct loc_network_list* list);
+ void loc_network_list_clear(struct loc_network_list* list);
++void loc_network_list_dump(struct loc_network_list* list);
+ struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
+ int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
+ struct loc_network* loc_network_list_pop(struct loc_network_list* list);
+diff --git a/src/network.c b/src/network.c
+index c9e7979..0977406 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -817,6 +817,20 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
+ list->size = 0;
+ }
+
++LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
++ struct loc_network* network;
++ char* s;
++
++ for (unsigned int i = 0; i < list->size; i++) {
++ network = list->list[i];
++
++ s = loc_network_str(network);
++
++ INFO(list->ctx, "%s\n", s);
++ free(s);
++ }
++}
++
+ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
+ // Check index
+ if (index >= list->size)
+--
+2.20.1
+
+From 850e75167e8e03fe8b951992c9f7bc2ccb9fb711 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 14:18:40 +0000
+Subject: [PATCH 35/70] network: Add functions to break network into subnets
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/libloc.sym | 5 +
+ src/loc/network.h | 6 +
+ src/network.c | 290 ++++++++++++++++++++++++++++++++++++++++++++-
+ src/test-network.c | 61 ++++++++++
+ 4 files changed, 361 insertions(+), 1 deletion(-)
+
+diff --git a/src/libloc.sym b/src/libloc.sym
+index a5641c6..fcb9ea5 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -80,6 +80,8 @@ global:
+
+ # Network
+ loc_network_address_family;
++ loc_network_eq;
++ loc_network_exclude;
+ loc_network_format_first_address;
+ loc_network_format_last_address;
+ loc_network_get_asn;
+@@ -96,6 +98,7 @@ global:
+ loc_network_set_country_code;
+ loc_network_set_flag;
+ loc_network_str;
++ loc_network_subnets;
+ loc_network_unref;
+
+ # Network List
+@@ -107,7 +110,9 @@ global:
+ loc_network_list_pop;
+ loc_network_list_push;
+ loc_network_list_ref;
++ loc_network_list_reverse;
+ loc_network_list_size;
++ loc_network_list_sort;
+ loc_network_list_unref;
+
+ # Writer
+diff --git a/src/loc/network.h b/src/loc/network.h
+index 44c50a4..4e51a62 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -54,7 +54,11 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
+ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
+ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
+
++int loc_network_eq(struct loc_network* self, struct loc_network* other);
+ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
++struct loc_network_list* loc_network_subnets(struct loc_network* network);
++struct loc_network_list* loc_network_exclude(
++ struct loc_network* self, struct loc_network* other);
+
+ // List
+ struct loc_network_list;
+@@ -68,6 +72,8 @@ void loc_network_list_dump(struct loc_network_list* list);
+ struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
+ int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
+ struct loc_network* loc_network_list_pop(struct loc_network_list* list);
++void loc_network_list_sort(struct loc_network_list* list);
++void loc_network_list_reverse(struct loc_network_list* list);
+
+ #ifdef LIBLOC_PRIVATE
+
+diff --git a/src/network.c b/src/network.c
+index 0977406..6c08070 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -97,6 +97,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
+ return a;
+ }
+
++static struct in6_addr address_increment(const struct in6_addr* address) {
++ struct in6_addr a = *address;
++
++ for (int octet = 15; octet >= 0; octet--) {
++ if (a.s6_addr[octet] < 255) {
++ a.s6_addr[octet]++;
++ break;
++ } else {
++ a.s6_addr[octet] = 0;
++ }
++ }
++
++ return a;
++}
++
+ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
+ struct in6_addr* address, unsigned int prefix) {
+ // Address cannot be unspecified
+@@ -405,6 +420,69 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
+ return loc_network_has_flag(network, flag);
+ }
+
++LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
++#ifdef ENABLE_DEBUG
++ char* n1 = loc_network_str(self);
++ char* n2 = loc_network_str(other);
++
++ DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
++
++ free(n1);
++ free(n2);
++#endif
++
++ // Family must be the same
++ if (self->family != other->family) {
++ DEBUG(self->ctx, " Family mismatch\n");
++
++ return 0;
++ }
++
++ // The start address must be the same
++ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
++ DEBUG(self->ctx, " Address mismatch\n");
++
++ return 0;
++ }
++
++ // The prefix length must be the same
++ if (self->prefix != other->prefix) {
++ DEBUG(self->ctx, " Prefix mismatch\n");
++ return 0;
++ }
++
++ DEBUG(self->ctx, " Yes!\n");
++
++ return 1;
++}
++
++static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
++ // Families must match
++ if (self->family != other->family)
++ return -1;
++
++ int r = in6_addr_cmp(&self->first_address, &other->first_address);
++
++ switch (r) {
++ // Smaller
++ case -1:
++ return 0;
++
++ // Larger
++ case 1:
++ return 1;
++
++ default:
++ break;
++ }
++
++ if (self->prefix > other->prefix)
++ return 1;
++
++ // Dunno
++ return 0;
++}
++
+ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
+ // If the start address of the other network is smaller than this network,
+ // it cannot be a subnet.
+@@ -419,6 +497,175 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net
+ return 1;
+ }
+
++LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
++ struct loc_network_list* list;
++
++ // New prefix length
++ unsigned int prefix = network->prefix + 1;
++
++ // Check if the new prefix is valid
++ if (valid_prefix(&network->first_address, prefix))
++ return NULL;
++
++ // Create a new list with the result
++ int r = loc_network_list_new(network->ctx, &list);
++ if (r) {
++ ERROR(network->ctx, "Could not create network list: %d\n", r);
++ return NULL;
++ }
++
++ struct loc_network* subnet1 = NULL;
++ struct loc_network* subnet2 = NULL;
++
++ // Create the first half of the network
++ r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix);
++ if (r)
++ goto ERROR;
++
++ // The next subnet starts after the first one
++ struct in6_addr first_address = address_increment(&subnet1->last_address);
++
++ // Create the second half of the network
++ r = loc_network_new(network->ctx, &subnet2, &first_address, prefix);
++ if (r)
++ goto ERROR;
++
++ // Push the both onto the stack (in reverse order)
++ r = loc_network_list_push(list, subnet2);
++ if (r)
++ goto ERROR;
++
++ r = loc_network_list_push(list, subnet1);
++ if (r)
++ goto ERROR;
++
++ loc_network_unref(subnet1);
++ loc_network_unref(subnet2);
++
++ return list;
++
++ERROR:
++ if (subnet1)
++ loc_network_unref(subnet1);
++
++ if (subnet2)
++ loc_network_unref(subnet2);
++
++ if (list)
++ loc_network_list_unref(list);
++
++ return NULL;
++}
++
++LOC_EXPORT struct loc_network_list* loc_network_exclude(
++ struct loc_network* self, struct loc_network* other) {
++ struct loc_network_list* list;
++
++#ifdef ENABLE_DEBUG
++ char* n1 = loc_network_str(self);
++ char* n2 = loc_network_str(other);
++
++ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
++
++ free(n1);
++ free(n2);
++#endif
++
++ // Family must match
++ if (self->family != other->family) {
++ DEBUG(self->ctx, "Family mismatch\n");
++
++ return NULL;
++ }
++
++ // Other must be a subnet of self
++ if (!loc_network_is_subnet_of(other, self)) {
++ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
++
++ return NULL;
++ }
++
++ // We cannot perform this operation if both networks equal
++ if (loc_network_eq(self, other)) {
++ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
++
++ return NULL;
++ }
++
++ // Create a new list with the result
++ int r = loc_network_list_new(self->ctx, &list);
++ if (r) {
++ ERROR(self->ctx, "Could not create network list: %d\n", r);
++ return NULL;
++ }
++
++ struct loc_network_list* subnets = loc_network_subnets(self);
++
++ struct loc_network* subnet1 = NULL;
++ struct loc_network* subnet2 = NULL;
++
++ while (subnets) {
++ // Fetch both subnets
++ subnet1 = loc_network_list_get(subnets, 0);
++ subnet2 = loc_network_list_get(subnets, 1);
++
++ // Free list
++ loc_network_list_unref(subnets);
++ subnets = NULL;
++
++ if (loc_network_eq(other, subnet1)) {
++ r = loc_network_list_push(list, subnet2);
++ if (r)
++ goto ERROR;
++
++ } else if (loc_network_eq(other, subnet2)) {
++ r = loc_network_list_push(list, subnet1);
++ if (r)
++ goto ERROR;
++
++ } else if (loc_network_is_subnet_of(other, subnet1)) {
++ r = loc_network_list_push(list, subnet2);
++ if (r)
++ goto ERROR;
++
++ subnets = loc_network_subnets(subnet1);
++
++ } else if (loc_network_is_subnet_of(other, subnet2)) {
++ r = loc_network_list_push(list, subnet1);
++ if (r)
++ goto ERROR;
++
++ subnets = loc_network_subnets(subnet2);
++
++ } else {
++ ERROR(self->ctx, "We should never get here\n");
++ goto ERROR;
++ }
++
++ loc_network_unref(subnet1);
++ loc_network_unref(subnet2);
++ }
++
++#ifdef ENABLE_DEBUG
++ loc_network_list_dump(list);
++#endif
++
++ // Return the result
++ return list;
++
++ERROR:
++ if (subnet1)
++ loc_network_unref(subnet1);
++
++ if (subnet2)
++ loc_network_unref(subnet2);
++
++ if (list)
++ loc_network_list_unref(list);
++
++ return NULL;
++}
++
+ LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
+ // Add country code
+ loc_country_code_copy(dbobj->country_code, network->country_code);
+@@ -854,5 +1101,46 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
+ if (loc_network_list_empty(list))
+ return NULL;
+
+- return list->list[list->size--];
++ return list->list[--list->size];
++}
++
++static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
++ // Do nothing for invalid indices
++ if (i1 >= list->size || i2 >= list->size)
++ return;
++
++ DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
++
++ struct loc_network* network1 = list->list[i1];
++ struct loc_network* network2 = list->list[i2];
++
++ list->list[i1] = network2;
++ list->list[i2] = network1;
++}
++
++LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
++ unsigned int i = 0;
++ unsigned int j = list->size - 1;
++
++ while (i < j) {
++ loc_network_list_swap(list, i++, j--);
++ }
++}
++
++LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
++ unsigned int n = list->size;
++ int swapped;
++
++ do {
++ swapped = 0;
++
++ for (unsigned int i = 1; i < n; i++) {
++ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
++ loc_network_list_swap(list, i-1, i);
++ swapped = 1;
++ }
++ }
++
++ n--;
++ } while (swapped);
+ }
+diff --git a/src/test-network.c b/src/test-network.c
+index b6776b4..af1b2e6 100644
+--- a/src/test-network.c
++++ b/src/test-network.c
+@@ -118,6 +118,19 @@ int main(int argc, char** argv) {
+ size_t nodes = loc_network_tree_count_nodes(tree);
+ printf("The tree has %zu nodes\n", nodes);
+
++ // Check equals function
++ err = loc_network_eq(network1, network1);
++ if (!err) {
++ fprintf(stderr, "Network is not equal with itself\n");
++ exit(EXIT_FAILURE);
++ }
++
++ err = loc_network_eq(network1, network2);
++ if (err) {
++ fprintf(stderr, "Networks equal unexpectedly\n");
++ exit(EXIT_FAILURE);
++ }
++
+ // Check subnet function
+ err = loc_network_is_subnet_of(network1, network2);
+ if (err != 0) {
+@@ -131,6 +144,54 @@ int main(int argc, char** argv) {
+ exit(EXIT_FAILURE);
+ }
+
++ // Make list of subnets
++ struct loc_network_list* subnets = loc_network_subnets(network1);
++ if (!subnets) {
++ fprintf(stderr, "Could not find subnets of network\n");
++ exit(EXIT_FAILURE);
++ }
++
++ loc_network_list_dump(subnets);
++
++ while (!loc_network_list_empty(subnets)) {
++ struct loc_network* subnet = loc_network_list_pop(subnets);
++ if (!subnet) {
++ fprintf(stderr, "Received an empty subnet\n");
++ exit(EXIT_FAILURE);
++ }
++
++ char* s = loc_network_str(subnet);
++ printf("Received subnet %s\n", s);
++ free(s);
++
++ if (!loc_network_is_subnet_of(subnet, network1)) {
++ fprintf(stderr, "Not a subnet\n");
++ exit(EXIT_FAILURE);
++ }
++
++ loc_network_unref(subnet);
++ }
++
++ loc_network_list_unref(subnets);
++
++ struct loc_network_list* excluded = loc_network_exclude(network1, network2);
++ if (!excluded) {
++ fprintf(stderr, "Could not create excluded list\n");
++ exit(EXIT_FAILURE);
++ }
++
++ loc_network_list_dump(excluded);
++
++ // Reverse it
++ loc_network_list_reverse(excluded);
++ loc_network_list_dump(excluded);
++
++ // Sort them and dump them again
++ loc_network_list_sort(excluded);
++ loc_network_list_dump(excluded);
++
++ loc_network_list_unref(excluded);
++
+ // Create a database
+ struct loc_writer* writer;
+ err = loc_writer_new(ctx, &writer, NULL, NULL);
+--
+2.20.1
+
+From 6159d384c4a98fe45ec52522e2950719e4982d80 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 14:24:58 +0000
+Subject: [PATCH 36/70] networks: Add function to check if two networks overlap
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/libloc.sym | 1 +
+ src/loc/network.h | 1 +
+ src/network.c | 16 ++++++++++++++++
+ 3 files changed, 18 insertions(+)
+
+diff --git a/src/libloc.sym b/src/libloc.sym
+index fcb9ea5..0c2835b 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -93,6 +93,7 @@ global:
+ loc_network_match_flag;
+ loc_network_new;
+ loc_network_new_from_string;
++ loc_network_overlaps;
+ loc_network_ref;
+ loc_network_set_asn;
+ loc_network_set_country_code;
+diff --git a/src/loc/network.h b/src/loc/network.h
+index 4e51a62..ef13756 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -55,6 +55,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
+ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
+
+ int loc_network_eq(struct loc_network* self, struct loc_network* other);
++int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
+ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
+ struct loc_network_list* loc_network_subnets(struct loc_network* network);
+ struct loc_network_list* loc_network_exclude(
+diff --git a/src/network.c b/src/network.c
+index 6c08070..d826511 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -483,6 +483,22 @@ static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
+ return 0;
+ }
+
++LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
++ if (loc_network_match_address(self, &other->first_address) == 0)
++ return 1;
++
++ if (loc_network_match_address(self, &other->last_address) == 0)
++ return 1;
++
++ if (loc_network_match_address(other, &self->first_address) == 0)
++ return 1;
++
++ if (loc_network_match_address(other, &self->last_address) == 0)
++ return 1;
++
++ return 0;
++}
++
+ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
+ // If the start address of the other network is smaller than this network,
+ // it cannot be a subnet.
+--
+2.20.1
+
+From e52ba21761f27e592040a2793b2a26bbeeeecc05 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 14:28:15 +0000
+Subject: [PATCH 37/70] networks: Add function to check if network is part of a
+ list
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/libloc.sym | 1 +
+ src/loc/network.h | 1 +
+ src/network.c | 9 +++++++++
+ 3 files changed, 11 insertions(+)
+
+diff --git a/src/libloc.sym b/src/libloc.sym
+index 0c2835b..f1b63a2 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -104,6 +104,7 @@ global:
+
+ # Network List
+ loc_network_list_clear;
++ loc_network_list_contains;
+ loc_network_list_dump;
+ loc_network_list_empty;
+ loc_network_list_get;
+diff --git a/src/loc/network.h b/src/loc/network.h
+index ef13756..7804512 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -73,6 +73,7 @@ void loc_network_list_dump(struct loc_network_list* list);
+ struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
+ int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
+ struct loc_network* loc_network_list_pop(struct loc_network_list* list);
++int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
+ void loc_network_list_sort(struct loc_network_list* list);
+ void loc_network_list_reverse(struct loc_network_list* list);
+
+diff --git a/src/network.c b/src/network.c
+index d826511..fcbdc59 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -1120,6 +1120,15 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
+ return list->list[--list->size];
+ }
+
++LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
++ for (unsigned int i = 0; i < list->size; i++) {
++ if (loc_network_eq(list->list[i], network))
++ return 1;
++ }
++
++ return 0;
++}
++
+ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
+ // Do nothing for invalid indices
+ if (i1 >= list->size || i2 >= list->size)
+--
+2.20.1
+
+From f802f3a4decf4827ecc8bcabe269ae9f94f7f32d Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 14:33:22 +0000
+Subject: [PATCH 38/70] networks: Add function to merge two lists
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/libloc.sym | 1 +
+ src/loc/network.h | 1 +
+ src/network.c | 13 +++++++++++++
+ 3 files changed, 15 insertions(+)
+
+diff --git a/src/libloc.sym b/src/libloc.sym
+index f1b63a2..c0b6b1f 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -108,6 +108,7 @@ global:
+ loc_network_list_dump;
+ loc_network_list_empty;
+ loc_network_list_get;
++ loc_network_list_merge;
+ loc_network_list_new;
+ loc_network_list_pop;
+ loc_network_list_push;
+diff --git a/src/loc/network.h b/src/loc/network.h
+index 7804512..e30d91c 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -76,6 +76,7 @@ struct loc_network* loc_network_list_pop(struct loc_network_list* list);
+ int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
+ void loc_network_list_sort(struct loc_network_list* list);
+ void loc_network_list_reverse(struct loc_network_list* list);
++int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
+
+ #ifdef LIBLOC_PRIVATE
+
+diff --git a/src/network.c b/src/network.c
+index fcbdc59..541286d 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -1169,3 +1169,16 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
+ n--;
+ } while (swapped);
+ }
++
++LOC_EXPORT int loc_network_list_merge(
++ struct loc_network_list* self, struct loc_network_list* other) {
++ int r;
++
++ for (unsigned int i = 0; i < other->size; i++) {
++ r = loc_network_list_push(self, other->list[i]);
++ if (r)
++ return r;
++ }
++
++ return 0;
++}
+--
+2.20.1
+
+From 6d22a179dffd08fcf2a44aafb361725ab22486d4 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 14:35:43 +0000
+Subject: [PATCH 39/70] network: Make lists unique
+
+Networks that are in the list won't be added again
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/network.c | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/src/network.c b/src/network.c
+index 541286d..44571b3 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -1103,6 +1103,10 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
+ }
+
+ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
++ // Do not add networks that are already on the list
++ if (loc_network_list_contains(list, network))
++ return 0;
++
+ // Check if we have space left
+ if (list->size == list->max_size)
+ return -ENOMEM;
+--
+2.20.1
+
+From 681ff05cb7cdf230d38abf09a330a31498e265a4 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 19:21:13 +0000
+Subject: [PATCH 40/70] database: Pass flag to enumerator to flatten output
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/database.c | 8 +++++++-
+ src/loc/database.h | 6 +++++-
+ src/perl/Location.xs | 2 +-
+ src/python/database.c | 25 ++++++++++++++++++-------
+ 4 files changed, 31 insertions(+), 10 deletions(-)
+
+diff --git a/src/database.c b/src/database.c
+index fa1dad0..9baab33 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -104,6 +104,9 @@ struct loc_database_enumerator {
+ enum loc_network_flags flags;
+ int family;
+
++ // Flatten output?
++ int flatten;
++
+ // Index of the AS we are looking at
+ unsigned int as_index;
+
+@@ -933,7 +936,7 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
+ // Enumerator
+
+ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
+- struct loc_database* db, enum loc_database_enumerator_mode mode) {
++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
+ struct loc_database_enumerator* e = calloc(1, sizeof(*e));
+ if (!e)
+ return -ENOMEM;
+@@ -944,6 +947,9 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
+ e->mode = mode;
+ e->refcount = 1;
+
++ // Flatten output?
++ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
++
+ // Initialise graph search
+ //e->network_stack[++e->network_stack_depth] = 0;
+ e->network_stack_depth = 1;
+diff --git a/src/loc/database.h b/src/loc/database.h
+index 43173dd..14eb5ea 100644
+--- a/src/loc/database.h
++++ b/src/loc/database.h
+@@ -55,9 +55,13 @@ enum loc_database_enumerator_mode {
+ LOC_DB_ENUMERATE_COUNTRIES = 3,
+ };
+
++enum loc_database_enumerator_flags {
++ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
++};
++
+ struct loc_database_enumerator;
+ int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
+- struct loc_database* db, enum loc_database_enumerator_mode mode);
++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
+ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
+ struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
+
+diff --git a/src/perl/Location.xs b/src/perl/Location.xs
+index dcf3f0d..b7676d2 100644
+--- a/src/perl/Location.xs
++++ b/src/perl/Location.xs
+@@ -125,7 +125,7 @@ database_countries(db)
+ PPCODE:
+ // Create Database enumerator
+ struct loc_database_enumerator* enumerator;
+- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
++ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
+
+ if (err) {
+ croak("Could not create a database enumerator\n");
+diff --git a/src/python/database.c b/src/python/database.c
+index 1013a58..7f8c2c2 100644
+--- a/src/python/database.c
++++ b/src/python/database.c
+@@ -207,10 +207,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
+ return (PyObject*)self;
+ }
+
+-static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
++static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
+ struct loc_database_enumerator* enumerator;
+
+- int r = loc_database_enumerator_new(&enumerator, self->db, what);
++ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
+ if (r) {
+ PyErr_SetFromErrno(PyExc_SystemError);
+ return NULL;
+@@ -223,7 +223,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
+ }
+
+ static PyObject* Database_ases(DatabaseObject* self) {
+- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
++ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
+ }
+
+ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
+@@ -234,7 +234,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
+
+ struct loc_database_enumerator* enumerator;
+
+- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
+ if (r) {
+ PyErr_SetFromErrno(PyExc_SystemError);
+ return NULL;
+@@ -250,7 +250,11 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
+ }
+
+ static PyObject* Database_networks(DatabaseObject* self) {
+- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
++}
++
++static PyObject* Database_networks_flattened(DatabaseObject *self) {
++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
+ }
+
+ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
+@@ -264,7 +268,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
+ return NULL;
+
+ struct loc_database_enumerator* enumerator;
+- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
+ if (r) {
+ PyErr_SetFromErrno(PyExc_SystemError);
+ return NULL;
+@@ -317,7 +321,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
+ }
+
+ static PyObject* Database_countries(DatabaseObject* self) {
+- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
++ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
+ }
+
+ static struct PyMethodDef Database_methods[] = {
+@@ -403,6 +407,13 @@ static struct PyGetSetDef Database_getsetters[] = {
+ NULL,
+ NULL,
+ },
++ {
++ "networks_flattened",
++ (getter)Database_networks_flattened,
++ NULL,
++ NULL,
++ NULL,
++ },
+ {
+ "vendor",
+ (getter)Database_get_vendor,
+--
+2.20.1
+
+From f5e50a47e37e9b29d0d2ee9e5a41e5a5fe5aea7f Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 19:21:58 +0000
+Subject: [PATCH 41/70] network: Reduce debugging output
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/network.c | 28 +++-------------------------
+ 1 file changed, 3 insertions(+), 25 deletions(-)
+
+diff --git a/src/network.c b/src/network.c
+index 44571b3..f7071a6 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -421,37 +421,17 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
+ }
+
+ LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
+-#ifdef ENABLE_DEBUG
+- char* n1 = loc_network_str(self);
+- char* n2 = loc_network_str(other);
+-
+- DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
+-
+- free(n1);
+- free(n2);
+-#endif
+-
+ // Family must be the same
+- if (self->family != other->family) {
+- DEBUG(self->ctx, " Family mismatch\n");
+-
++ if (self->family != other->family)
+ return 0;
+- }
+
+ // The start address must be the same
+- if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
+- DEBUG(self->ctx, " Address mismatch\n");
+-
++ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0)
+ return 0;
+- }
+
+ // The prefix length must be the same
+- if (self->prefix != other->prefix) {
+- DEBUG(self->ctx, " Prefix mismatch\n");
++ if (self->prefix != other->prefix)
+ return 0;
+- }
+-
+- DEBUG(self->ctx, " Yes!\n");
+
+ return 1;
+ }
+@@ -1138,8 +1118,6 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
+ if (i1 >= list->size || i2 >= list->size)
+ return;
+
+- DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
+-
+ struct loc_network* network1 = list->list[i1];
+ struct loc_network* network2 = list->list[i2];
+
+--
+2.20.1
+
+From 037c65d3a07ec6d37ff063f0645adda6b483b407 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 19:36:38 +0000
+Subject: [PATCH 42/70] python: Export networks exclude function
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/network.c | 39 +++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 39 insertions(+)
+
+diff --git a/src/python/network.c b/src/python/network.c
+index 5496d1e..11f672b 100644
+--- a/src/python/network.c
++++ b/src/python/network.c
+@@ -24,6 +24,24 @@
+ #include "locationmodule.h"
+ #include "network.h"
+
++static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
++ PyObject* list = PyList_New(0);
++ if (!networks)
++ return list;
++
++ while (!loc_network_list_empty(networks)) {
++ struct loc_network* network = loc_network_list_pop(networks);
++
++ PyObject* n = new_network(&NetworkType, network);
++ PyList_Append(list, n);
++
++ loc_network_unref(network);
++ Py_DECREF(n);
++ }
++
++ return list;
++}
++
+ PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
+ NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
+ if (self) {
+@@ -154,6 +172,21 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
+ Py_RETURN_NONE;
+ }
+
++static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
++ NetworkObject* other = NULL;
++
++ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
++ return NULL;
++
++ struct loc_network_list* list = loc_network_exclude(self->network, other->network);
++
++ // Convert to Python objects
++ PyObject* obj = PyList_FromNetworkList(list);
++ loc_network_list_unref(list);
++
++ return obj;
++}
++
+ static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
+ NetworkObject* other = NULL;
+
+@@ -191,6 +224,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
+ }
+
+ static struct PyMethodDef Network_methods[] = {
++ {
++ "exclude",
++ (PyCFunction)Network_exclude,
++ METH_VARARGS,
++ NULL,
++ },
+ {
+ "has_flag",
+ (PyCFunction)Network_has_flag,
+--
+2.20.1
+
+From 9a7732c8679e805d4d2d55ea4750c5d70ca4bd2c Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 19:59:22 +0000
+Subject: [PATCH 43/70] network: Add more debugging output to stacks
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/network.c | 16 +++++++++++++---
+ 1 file changed, 13 insertions(+), 3 deletions(-)
+
+diff --git a/src/network.c b/src/network.c
+index f7071a6..d41e873 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -1088,8 +1088,12 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
+ return 0;
+
+ // Check if we have space left
+- if (list->size == list->max_size)
++ if (list->size == list->max_size) {
++ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
+ return -ENOMEM;
++ }
++
++ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
+
+ list->list[list->size++] = loc_network_ref(network);
+
+@@ -1098,10 +1102,16 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
+
+ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
+ // Return nothing when empty
+- if (loc_network_list_empty(list))
++ if (loc_network_list_empty(list)) {
++ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
+ return NULL;
++ }
+
+- return list->list[--list->size];
++ struct loc_network* network = list->list[--list->size];
++
++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
++
++ return network;
+ }
+
+ LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
+--
+2.20.1
+
+From 33a051e0435f6e78cc936f26f3b9ee16b7851025 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 20:00:09 +0000
+Subject: [PATCH 44/70] network: Add new subnet function
+
+The old one is too difficult to use in terms of order
+of input parameters and return value.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/libloc.sym | 1 +
+ src/loc/network.h | 1 +
+ src/network.c | 15 +++++++++++++++
+ 3 files changed, 17 insertions(+)
+
+diff --git a/src/libloc.sym b/src/libloc.sym
+index c0b6b1f..5392437 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -87,6 +87,7 @@ global:
+ loc_network_get_asn;
+ loc_network_get_country_code;
+ loc_network_has_flag;
++ loc_network_is_subnet;
+ loc_network_is_subnet_of;
+ loc_network_match_asn;
+ loc_network_match_country_code;
+diff --git a/src/loc/network.h b/src/loc/network.h
+index e30d91c..2154cdc 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -56,6 +56,7 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
+
+ int loc_network_eq(struct loc_network* self, struct loc_network* other);
+ int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
++int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
+ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
+ struct loc_network_list* loc_network_subnets(struct loc_network* network);
+ struct loc_network_list* loc_network_exclude(
+diff --git a/src/network.c b/src/network.c
+index d41e873..5719111 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -479,6 +479,21 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
+ return 0;
+ }
+
++LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
++ // If the start address of the other network is smaller than this network,
++ // it cannot be a subnet.
++ if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
++ return 0;
++
++ // If the end address of the other network is greater than this network,
++ // it cannot be a subnet.
++ if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
++ return 0;
++
++ return 1;
++}
++
++// XXX DEPRECATED - I find this too difficult to use
+ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
+ // If the start address of the other network is smaller than this network,
+ // it cannot be a subnet.
+--
+2.20.1
+
+From add5bb652ba1dad1127f79cb6a0db2d363a6d5e5 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 20:01:17 +0000
+Subject: [PATCH 45/70] network: Add function to exclude multiple networks at
+ once
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/libloc.sym | 1 +
+ src/loc/network.h | 2 ++
+ src/network.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++
+ 3 files changed, 88 insertions(+)
+
+diff --git a/src/libloc.sym b/src/libloc.sym
+index 5392437..bcd11be 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -82,6 +82,7 @@ global:
+ loc_network_address_family;
+ loc_network_eq;
+ loc_network_exclude;
++ loc_network_exclude_list;
+ loc_network_format_first_address;
+ loc_network_format_last_address;
+ loc_network_get_asn;
+diff --git a/src/loc/network.h b/src/loc/network.h
+index 2154cdc..40712b9 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -61,6 +61,8 @@ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other
+ struct loc_network_list* loc_network_subnets(struct loc_network* network);
+ struct loc_network_list* loc_network_exclude(
+ struct loc_network* self, struct loc_network* other);
++struct loc_network_list* loc_network_exclude_list(
++ struct loc_network* network, struct loc_network_list* list);
+
+ // List
+ struct loc_network_list;
+diff --git a/src/network.c b/src/network.c
+index 5719111..751e8e5 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -677,6 +677,91 @@ ERROR:
+ return NULL;
+ }
+
++LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
++ struct loc_network* network, struct loc_network_list* list) {
++ struct loc_network_list* to_check;
++
++ // Create a new list with all networks to look at
++ int r = loc_network_list_new(network->ctx, &to_check);
++ if (r)
++ return NULL;
++
++ struct loc_network* subnet = NULL;
++ struct loc_network_list* subnets = NULL;
++
++ for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
++ subnet = loc_network_list_get(list, i);
++
++ // Find all excluded networks
++ struct loc_network_list* excluded = loc_network_exclude(network, subnet);
++ if (excluded) {
++ // Add them all to the "to check" list
++ loc_network_list_merge(to_check, excluded);
++ loc_network_list_unref(excluded);
++ }
++
++ // Cleanup
++ loc_network_unref(subnet);
++ }
++
++ r = loc_network_list_new(network->ctx, &subnets);
++ if (r) {
++ loc_network_list_unref(to_check);
++ return NULL;
++ }
++
++ while (!loc_network_list_empty(to_check)) {
++ struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
++
++ // Marks whether this subnet passed all checks
++ int passed = 1;
++
++ for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
++ subnet = loc_network_list_get(list, i);
++
++ // Drop this subnet if is is already in list
++ if (loc_network_eq(subnet_to_check, subnet)) {
++ passed = 0;
++ loc_network_unref(subnet);
++ break;
++ }
++
++ // Drop this subnet if is a subnet of another subnet
++ if (loc_network_is_subnet_of(subnet, subnet_to_check)) {
++ passed = 0;
++ loc_network_unref(subnet);
++ break;
++ }
++
++ // Break it down if it overlaps
++ if (loc_network_overlaps(subnet_to_check, subnet)) {
++ passed = 0;
++
++ struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet);
++ if (excluded) {
++ loc_network_list_merge(to_check, excluded);
++ loc_network_list_unref(excluded);
++ }
++
++ loc_network_unref(subnet);
++ break;
++ }
++
++ loc_network_unref(subnet);
++ }
++
++ if (passed) {
++ r = loc_network_list_push(subnets, subnet_to_check);
++ }
++
++ loc_network_unref(subnet_to_check);
++ }
++
++ loc_network_list_unref(to_check);
++
++ return subnets;
++}
++
+ LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
+ // Add country code
+ loc_country_code_copy(dbobj->country_code, network->country_code);
+--
+2.20.1
+
+From d87fd7a3d277b4b03222c7d1680e51b3e45e525b Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 20:02:03 +0000
+Subject: [PATCH 46/70] database: Add option to return networks flattened
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/database.c | 174 ++++++++++++++++++++++++++++++++++++++++---------
+ 1 file changed, 143 insertions(+), 31 deletions(-)
+
+diff --git a/src/database.c b/src/database.c
+index 9baab33..7a3d1a7 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -118,6 +118,9 @@ struct loc_database_enumerator {
+ struct loc_node_stack network_stack[MAX_STACK_DEPTH];
+ int network_stack_depth;
+ unsigned int* networks_visited;
++
++ // For subnet search
++ struct loc_network_list* stack;
+ };
+
+ static int loc_database_read_magic(struct loc_database* db) {
+@@ -935,6 +938,26 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
+
+ // Enumerator
+
++static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
++ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
++
++ // Release all references
++ loc_database_unref(enumerator->db);
++ loc_unref(enumerator->ctx);
++
++ if (enumerator->string)
++ free(enumerator->string);
++
++ // Free network search
++ free(enumerator->networks_visited);
++
++ // Free subnet stack
++ if (enumerator->stack)
++ loc_network_list_unref(enumerator->stack);
++
++ free(enumerator);
++}
++
+ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
+ struct loc_database_enumerator* e = calloc(1, sizeof(*e));
+@@ -951,10 +974,16 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
+ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
+
+ // Initialise graph search
+- //e->network_stack[++e->network_stack_depth] = 0;
+ e->network_stack_depth = 1;
+ e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
+
++ // Allocate stack
++ int r = loc_network_list_new(e->ctx, &e->stack);
++ if (r) {
++ loc_database_enumerator_free(e);
++ return r;
++ }
++
+ DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
+
+ *enumerator = e;
+@@ -967,22 +996,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
+ return enumerator;
+ }
+
+-static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
+- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
+-
+- // Release all references
+- loc_database_unref(enumerator->db);
+- loc_unref(enumerator->ctx);
+-
+- if (enumerator->string)
+- free(enumerator->string);
+-
+- // Free network search
+- free(enumerator->networks_visited);
+-
+- free(enumerator);
+-}
+-
+ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
+ if (!enumerator)
+ return NULL;
+@@ -1116,17 +1129,13 @@ static int loc_database_enumerator_stack_push_node(
+ return 0;
+ }
+
+-LOC_EXPORT int loc_database_enumerator_next_network(
+- struct loc_database_enumerator* enumerator, struct loc_network** network) {
+- // Reset network
+- *network = NULL;
+-
+- // Do not do anything if not in network mode
+- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
++static int __loc_database_enumerator_next_network(
++ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
++ // Return top element from the stack
++ *network = loc_network_list_pop(enumerator->stack);
++ if (*network)
+ return 0;
+
+- int r;
+-
+ DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
+ enumerator->network_stack_depth);
+
+@@ -1155,7 +1164,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
+ enumerator->db->network_nodes_v1 + node->offset;
+
+ // Add edges to stack
+- r = loc_database_enumerator_stack_push_node(enumerator,
++ int r = loc_database_enumerator_stack_push_node(enumerator,
+ be32toh(n->one), 1, node->depth + 1);
+
+ if (r)
+@@ -1181,6 +1190,10 @@ LOC_EXPORT int loc_database_enumerator_next_network(
+ if (r)
+ return r;
+
++ // Return all networks when the filter is disabled
++ if (!filter)
++ return 0;
++
+ // Check if we are interested in this network
+
+ // Skip if the family does not match
+@@ -1223,12 +1236,111 @@ LOC_EXPORT int loc_database_enumerator_next_network(
+ }
+
+ // Reached the end of the search
++ return 0;
++}
+
+- // Mark all nodes as non-visited
+- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
+- enumerator->networks_visited[i] = 0;
++static int __loc_database_enumerator_next_network_flattened(
++ struct loc_database_enumerator* enumerator, struct loc_network** network) {
++ // Fetch the next network
++ int r = __loc_database_enumerator_next_network(enumerator, network, 1);
++ if (r)
++ return r;
+
+- return 0;
++ // End if we could not read another network
++ if (!*network)
++ return 0;
++
++ struct loc_network* subnet = NULL;
++ struct loc_network_list* subnets;
++
++ // Create a list with all subnets
++ r = loc_network_list_new(enumerator->ctx, &subnets);
++ if (r)
++ return r;
++
++ // Search all subnets from the database
++ while (1) {
++ // Fetch the next network in line
++ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
++ if (r)
++ goto END;
++
++ // End if we did not receive another subnet
++ if (!subnet)
++ break;
++
++ // Collect all subnets in a list
++ if (loc_network_is_subnet(*network, subnet)) {
++ r = loc_network_list_push(subnets, subnet);
++ if (r)
++ goto END;
++
++ loc_network_unref(subnet);
++ continue;
++ }
++
++ // If this is not a subnet, we push it back onto the stack and break
++ r = loc_network_list_push(enumerator->stack, subnet);
++ if (r)
++ goto END;
++
++ loc_network_unref(subnet);
++ break;
++ }
++
++ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
++
++ // We can abort here if the network has no subnets
++ if (loc_network_list_empty(subnets)) {
++ loc_network_list_unref(subnets);
++
++ return 0;
++ }
++
++ // If the network has any subnets, we will break it into smaller parts
++ // without the subnets.
++ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
++ if (!excluded || loc_network_list_empty(excluded)) {
++ r = 1;
++ goto END;
++ }
++
++ // Sort the result
++ loc_network_list_sort(excluded);
++
++ // Reverse the list
++ loc_network_list_reverse(excluded);
++
++ // Replace network with the first one
++ loc_network_unref(*network);
++
++ *network = loc_network_list_pop(excluded);
++
++ // Push the rest onto the stack
++ loc_network_list_merge(enumerator->stack, excluded);
++
++ loc_network_list_unref(excluded);
++
++END:
++ if (subnet)
++ loc_network_unref(subnet);
++
++ loc_network_list_unref(subnets);
++
++ return r;
++}
++
++LOC_EXPORT int loc_database_enumerator_next_network(
++ struct loc_database_enumerator* enumerator, struct loc_network** network) {
++ // Do not do anything if not in network mode
++ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
++ return 0;
++
++ // Flatten output?
++ if (enumerator->flatten)
++ return __loc_database_enumerator_next_network_flattened(enumerator, network);
++
++ return __loc_database_enumerator_next_network(enumerator, network, 1);
+ }
+
+ LOC_EXPORT int loc_database_enumerator_next_country(
+--
+2.20.1
+
+From d3ae93c27dcd7f6984fdc29cc141621e277f2e2a Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 20:09:20 +0000
+Subject: [PATCH 47/70] test: Update API
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/test-as.c | 2 +-
+ src/test-database.c | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/test-as.c b/src/test-as.c
+index 839a04c..2d61675 100644
+--- a/src/test-as.c
++++ b/src/test-as.c
+@@ -95,7 +95,7 @@ int main(int argc, char** argv) {
+ // Enumerator
+
+ struct loc_database_enumerator* enumerator;
+- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
+ if (err) {
+ fprintf(stderr, "Could not create a database enumerator\n");
+ exit(EXIT_FAILURE);
+diff --git a/src/test-database.c b/src/test-database.c
+index 4aef94e..da4b11c 100644
+--- a/src/test-database.c
++++ b/src/test-database.c
+@@ -198,7 +198,7 @@ int main(int argc, char** argv) {
+
+ // Enumerator
+ struct loc_database_enumerator* enumerator;
+- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
+ if (err) {
+ fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
+ exit(EXIT_FAILURE);
+--
+2.20.1
+
+From 594ca328c6e124d0f1eb543e9c8d9bbfe8a7b628 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 12 Nov 2020 20:09:37 +0000
+Subject: [PATCH 48/70] networks: Copy all attributes when splitting networks
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/network.c | 14 ++++++++++++++
+ 1 file changed, 14 insertions(+)
+
+diff --git a/src/network.c b/src/network.c
+index 751e8e5..d67f116 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -550,6 +550,20 @@ LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* netw
+ if (r)
+ goto ERROR;
+
++ // Copy country code
++ const char* country_code = loc_network_get_country_code(network);
++ if (country_code) {
++ loc_network_set_country_code(subnet1, country_code);
++ loc_network_set_country_code(subnet2, country_code);
++ }
++
++ // Copy ASN
++ uint32_t asn = loc_network_get_asn(network);
++ if (asn) {
++ loc_network_set_asn(subnet1, asn);
++ loc_network_set_asn(subnet2, asn);
++ }
++
+ loc_network_unref(subnet1);
+ loc_network_unref(subnet2);
+
+--
+2.20.1
+
+From 69248038292e9ea1a4ee8912cdfc8700456753ad Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri, 13 Nov 2020 11:23:33 +0000
+Subject: [PATCH 49/70] database: Move network filtering into a separate
+ function
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/database.c | 56 +++++++++++++++++++++++---------------------------
+ 1 file changed, 26 insertions(+), 30 deletions(-)
+
+diff --git a/src/database.c b/src/database.c
+index 7a3d1a7..72bc8eb 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -1129,6 +1129,31 @@ static int loc_database_enumerator_stack_push_node(
+ return 0;
+ }
+
++static int loc_database_enumerator_filter_network(
++ struct loc_database_enumerator* enumerator, struct loc_network* network) {
++ // Skip if the family does not match
++ if (enumerator->family && loc_network_address_family(network) != enumerator->family)
++ return 1;
++
++ // Skip if the country code does not match
++ if (*enumerator->country_code &&
++ !loc_network_match_country_code(network, enumerator->country_code))
++ return 1;
++
++ // Skip if the ASN does not match
++ if (enumerator->asn &&
++ !loc_network_match_asn(network, enumerator->asn))
++ return 1;
++
++ // Skip if flags do not match
++ if (enumerator->flags &&
++ !loc_network_match_flag(network, enumerator->flags))
++ return 1;
++
++ // Do not filter
++ return 0;
++}
++
+ static int __loc_database_enumerator_next_network(
+ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
+ // Return top element from the stack
+@@ -1195,36 +1220,7 @@ static int __loc_database_enumerator_next_network(
+ return 0;
+
+ // Check if we are interested in this network
+-
+- // Skip if the family does not match
+- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
+- loc_network_unref(*network);
+- *network = NULL;
+-
+- continue;
+- }
+-
+- // Skip if the country code does not match
+- if (*enumerator->country_code &&
+- !loc_network_match_country_code(*network, enumerator->country_code)) {
+- loc_network_unref(*network);
+- *network = NULL;
+-
+- continue;
+- }
+-
+- // Skip if the ASN does not match
+- if (enumerator->asn &&
+- !loc_network_match_asn(*network, enumerator->asn)) {
+- loc_network_unref(*network);
+- *network = NULL;
+-
+- continue;
+- }
+-
+- // Skip if flags do not match
+- if (enumerator->flags &&
+- !loc_network_match_flag(*network, enumerator->flags)) {
++ if (loc_database_enumerator_filter_network(enumerator, *network)) {
+ loc_network_unref(*network);
+ *network = NULL;
+
+--
+2.20.1
+
+From 2113e71bf7b997c82670c5c22cf91aa6442fe6f3 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri, 13 Nov 2020 11:29:02 +0000
+Subject: [PATCH 50/70] database: Filter results coming from stack
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/database.c | 18 ++++++++++++++++--
+ 1 file changed, 16 insertions(+), 2 deletions(-)
+
+diff --git a/src/database.c b/src/database.c
+index 72bc8eb..0f3cdc2 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -1157,9 +1157,23 @@ static int loc_database_enumerator_filter_network(
+ static int __loc_database_enumerator_next_network(
+ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
+ // Return top element from the stack
+- *network = loc_network_list_pop(enumerator->stack);
+- if (*network)
++ while (1) {
++ *network = loc_network_list_pop(enumerator->stack);
++
++ // Stack is empty
++ if (!*network)
++ break;
++
++ // Throw away any networks by filter
++ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
++ loc_network_unref(*network);
++ *network = NULL;
++ continue;
++ }
++
++ // Return result
+ return 0;
++ }
+
+ DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
+ enumerator->network_stack_depth);
+--
+2.20.1
+
+From d33753688138c9938743dafbbdddf220dd2afd14 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri, 13 Nov 2020 11:29:15 +0000
+Subject: [PATCH 51/70] network: Sort result of excluded lists
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/database.c | 3 ---
+ src/network.c | 3 +++
+ 2 files changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/src/database.c b/src/database.c
+index 0f3cdc2..6849d97 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -1315,9 +1315,6 @@ static int __loc_database_enumerator_next_network_flattened(
+ goto END;
+ }
+
+- // Sort the result
+- loc_network_list_sort(excluded);
+-
+ // Reverse the list
+ loc_network_list_reverse(excluded);
+
+diff --git a/src/network.c b/src/network.c
+index d67f116..9d02bf8 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -773,6 +773,9 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
+
+ loc_network_list_unref(to_check);
+
++ // Sort the result
++ loc_network_list_sort(subnets);
++
+ return subnets;
+ }
+
+--
+2.20.1
+
+From 8d777f12f7ffa4df1b28d197563888296803b727 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri, 13 Nov 2020 11:38:15 +0000
+Subject: [PATCH 52/70] network: Add function to pop first element from stack
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/database.c | 6 ++----
+ src/libloc.sym | 1 +
+ src/loc/network.h | 1 +
+ src/network.c | 19 +++++++++++++++++++
+ 4 files changed, 23 insertions(+), 4 deletions(-)
+
+diff --git a/src/database.c b/src/database.c
+index 6849d97..b9d870f 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -1315,15 +1315,13 @@ static int __loc_database_enumerator_next_network_flattened(
+ goto END;
+ }
+
+- // Reverse the list
+- loc_network_list_reverse(excluded);
+-
+ // Replace network with the first one
+ loc_network_unref(*network);
+
+- *network = loc_network_list_pop(excluded);
++ *network = loc_network_list_pop_first(excluded);
+
+ // Push the rest onto the stack
++ loc_network_list_reverse(excluded);
+ loc_network_list_merge(enumerator->stack, excluded);
+
+ loc_network_list_unref(excluded);
+diff --git a/src/libloc.sym b/src/libloc.sym
+index bcd11be..6139db6 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -113,6 +113,7 @@ global:
+ loc_network_list_merge;
+ loc_network_list_new;
+ loc_network_list_pop;
++ loc_network_list_pop_first;
+ loc_network_list_push;
+ loc_network_list_ref;
+ loc_network_list_reverse;
+diff --git a/src/loc/network.h b/src/loc/network.h
+index 40712b9..203e61c 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -76,6 +76,7 @@ void loc_network_list_dump(struct loc_network_list* list);
+ struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
+ int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
+ struct loc_network* loc_network_list_pop(struct loc_network_list* list);
++struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
+ int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
+ void loc_network_list_sort(struct loc_network_list* list);
+ void loc_network_list_reverse(struct loc_network_list* list);
+diff --git a/src/network.c b/src/network.c
+index 9d02bf8..e7dc97e 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -1231,6 +1231,25 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
+ return network;
+ }
+
++LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
++ // Return nothing when empty
++ if (loc_network_list_empty(list)) {
++ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
++ return NULL;
++ }
++
++ struct loc_network* network = list->list[0];
++
++ // Move all elements to the top of the stack
++ for (unsigned int i = 0; i < --list->size; i++) {
++ list->list[i] = list->list[i+1];
++ }
++
++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
++
++ return network;
++}
++
+ LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
+ for (unsigned int i = 0; i < list->size; i++) {
+ if (loc_network_eq(list->list[i], network))
+--
+2.20.1
+
+From 7933f5bfb4dd7603cb646e192840762bf6394292 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri, 13 Nov 2020 11:43:53 +0000
+Subject: [PATCH 53/70] network: Unexport all tree functions
+
+These should not be exported
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/network.c | 28 ++++++++++++++--------------
+ src/test-network.c | 9 +++++++++
+ 2 files changed, 23 insertions(+), 14 deletions(-)
+
+diff --git a/src/network.c b/src/network.c
+index e7dc97e..d015579 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -847,7 +847,7 @@ struct loc_network_tree_node {
+ struct loc_network* network;
+ };
+
+-LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
++int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
+ struct loc_network_tree* t = calloc(1, sizeof(*t));
+ if (!t)
+ return -ENOMEM;
+@@ -867,7 +867,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
+ return 0;
+ }
+
+-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
++struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
+ return loc_network_tree_node_ref(tree->root);
+ }
+
+@@ -939,7 +939,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
+ return 0;
+ }
+
+-LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
++int loc_network_tree_walk(struct loc_network_tree* tree,
+ int(*filter_callback)(struct loc_network* network, void* data),
+ int(*callback)(struct loc_network* network, void* data), void* data) {
+ return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
+@@ -954,7 +954,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
+ free(tree);
+ }
+
+-LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
++struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
+ if (--tree->refcount > 0)
+ return tree;
+
+@@ -975,13 +975,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
+ return 0;
+ }
+
+-LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
++int loc_network_tree_dump(struct loc_network_tree* tree) {
+ DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
+
+ return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
+ }
+
+-LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
++int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
+ DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
+
+ struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
+@@ -1012,7 +1012,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
+ return 0;
+ }
+
+-LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
++size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
+ size_t counter = 0;
+
+ int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
+@@ -1034,11 +1034,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
+ return counter;
+ }
+
+-LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
++size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
+ return __loc_network_tree_count_nodes(tree->root);
+ }
+
+-LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
++int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
+ struct loc_network_tree_node* n = calloc(1, sizeof(*n));
+ if (!n)
+ return -ENOMEM;
+@@ -1053,7 +1053,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
+ return 0;
+ }
+
+-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
++struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
+ if (node)
+ node->refcount++;
+
+@@ -1076,7 +1076,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
+ free(node);
+ }
+
+-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
++struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
+ if (!node)
+ return NULL;
+
+@@ -1087,7 +1087,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
+ return NULL;
+ }
+
+-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
++struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
+ if (index == 0)
+ node = node->zero;
+ else
+@@ -1099,11 +1099,11 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
+ return loc_network_tree_node_ref(node);
+ }
+
+-LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
++int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
+ return (!!node->network);
+ }
+
+-LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
++struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
+ return loc_network_ref(node->network);
+ }
+
+diff --git a/src/test-network.c b/src/test-network.c
+index af1b2e6..7c90224 100644
+--- a/src/test-network.c
++++ b/src/test-network.c
+@@ -37,12 +37,14 @@ int main(int argc, char** argv) {
+ // Enable debug logging
+ loc_set_log_priority(ctx, LOG_DEBUG);
+
++#if 0
+ struct loc_network_tree* tree;
+ err = loc_network_tree_new(ctx, &tree);
+ if (err) {
+ fprintf(stderr, "Could not create the network tree\n");
+ exit(EXIT_FAILURE);
+ }
++#endif
+
+ // Create a network
+ struct loc_network* network1;
+@@ -58,12 +60,14 @@ int main(int argc, char** argv) {
+ exit(EXIT_FAILURE);
+ }
+
++#if 0
+ // Adding network to the tree
+ err = loc_network_tree_add_network(tree, network1);
+ if (err) {
+ fprintf(stderr, "Could not add network to the tree\n");
+ exit(EXIT_FAILURE);
+ }
++#endif
+
+ // Check if the first and last addresses are correct
+ char* string = loc_network_format_first_address(network1);
+@@ -101,6 +105,7 @@ int main(int argc, char** argv) {
+ exit(EXIT_FAILURE);
+ }
+
++#if 0
+ // Adding network to the tree
+ err = loc_network_tree_add_network(tree, network2);
+ if (err) {
+@@ -117,6 +122,7 @@ int main(int argc, char** argv) {
+
+ size_t nodes = loc_network_tree_count_nodes(tree);
+ printf("The tree has %zu nodes\n", nodes);
++#endif
+
+ // Check equals function
+ err = loc_network_eq(network1, network1);
+@@ -260,7 +266,10 @@ int main(int argc, char** argv) {
+ loc_network_unref(network2);
+ loc_network_unref(network3);
+ loc_network_unref(network4);
++
++#if 0
+ loc_network_tree_unref(tree);
++#endif
+
+ // And open it again from disk
+ struct loc_database* db;
+--
+2.20.1
+
+From c242f7325bd6fc4ba26047ac24196d1c161c6e01 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri, 13 Nov 2020 12:09:03 +0000
+Subject: [PATCH 54/70] python: Move tree flattening into C
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/database.c | 8 ++-
+ src/python/export.py | 138 +++++-------------------------------------
+ 2 files changed, 21 insertions(+), 125 deletions(-)
+
+diff --git a/src/python/database.c b/src/python/database.c
+index 7f8c2c2..d169547 100644
+--- a/src/python/database.c
++++ b/src/python/database.c
+@@ -258,17 +258,19 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
+ }
+
+ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
+- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
++ char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
+ const char* country_code = NULL;
+ unsigned int asn = 0;
+ int flags = 0;
+ int family = 0;
++ int flatten = 0;
+
+- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
+ return NULL;
+
+ struct loc_database_enumerator* enumerator;
+- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
++ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
+ if (r) {
+ PyErr_SetFromErrno(PyExc_SystemError);
+ return NULL;
+diff --git a/src/python/export.py b/src/python/export.py
+index dd44332..be4a68e 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -87,58 +87,12 @@ class OutputWriter(object):
+ def _write_network(self, network):
+ self.f.write("%s\n" % network)
+
+- def write(self, network, subnets):
++ def write(self, network):
+ if self.flatten and self._flatten(network):
+ log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
+ return
+
+- # Convert network into a Python object
+- _network = ipaddress.ip_network("%s" % network)
+-
+- # Write the network when it has no subnets
+- if not subnets:
+- log.debug("Writing %s to %s" % (_network, self.f))
+- return self._write_network(_network)
+-
+- # Convert subnets into Python objects
+- _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
+-
+- # Split the network into smaller bits so that
+- # we can accomodate for any gaps in it later
+- to_check = set()
+- for _subnet in _subnets:
+- to_check.update(
+- _network.address_exclude(_subnet)
+- )
+-
+- # Clear the list of all subnets
+- subnets = []
+-
+- # Check if all subnets to not overlap with anything else
+- while to_check:
+- subnet_to_check = to_check.pop()
+-
+- for _subnet in _subnets:
+- # Drop this subnet if it equals one of the subnets
+- # or if it is subnet of one of them
+- if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
+- break
+-
+- # Break it down if it overlaps
+- if subnet_to_check.overlaps(_subnet):
+- to_check.update(
+- subnet_to_check.address_exclude(_subnet)
+- )
+- break
+-
+- # Add the subnet again as it passed the check
+- else:
+- subnets.append(subnet_to_check)
+-
+- # Write all networks as compact as possible
+- for network in ipaddress.collapse_addresses(subnets):
+- log.debug("Writing %s to %s" % (network, self.f))
+- self._write_network(network)
++ return self._write_network(network)
+
+ def finish(self):
+ """
+@@ -188,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter):
+ mode = "wb"
+
+ def _write_network(self, network):
+- for address in (network.network_address, network.broadcast_address):
++ for address in (network.first_address, network.last_address):
+ # Convert this into a string of bits
+ bytes = socket.inet_pton(
+ socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
+@@ -231,42 +185,21 @@ class Exporter(object):
+ writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
+
+ # Get all networks that match the family
+- networks = self.db.search_networks(family=family)
+-
+- # Create a stack with all networks in order where we can put items back
+- # again and retrieve them in the next iteration.
+- networks = BufferedStack(networks)
++ networks = self.db.search_networks(family=family, flatten=True)
+
+ # Walk through all networks
+ for network in networks:
+- # Collect all networks which are a subnet of network
+- subnets = []
+- for subnet in networks:
+- # If the next subnet was not a subnet, we have to push
+- # it back on the stack and break this loop
+- if not subnet.is_subnet_of(network):
+- networks.push(subnet)
+- break
+-
+- subnets.append(subnet)
+-
+ # Write matching countries
+- if network.country_code and network.country_code in writers:
+- # Mismatching subnets
+- gaps = [
+- subnet for subnet in subnets if not network.country_code == subnet.country_code
+- ]
+-
+- writers[network.country_code].write(network, gaps)
++ try:
++ writers[network.country_code].write(network)
++ except KeyError:
++ pass
+
+ # Write matching ASNs
+- if network.asn and network.asn in writers:
+- # Mismatching subnets
+- gaps = [
+- subnet for subnet in subnets if not network.asn == subnet.asn
+- ]
+-
+- writers[network.asn].write(network, gaps)
++ try:
++ writers[network.asn].write(network)
++ except KeyError:
++ pass
+
+ # Handle flags
+ for flag in flags:
+@@ -274,19 +207,10 @@ class Exporter(object):
+ # Fetch the "fake" country code
+ country = flags[flag]
+
+- if not country in writers:
+- continue
+-
+- gaps = [
+- subnet for subnet in subnets
+- if not subnet.has_flag(flag)
+- ]
+-
+- writers[country].write(network, gaps)
+-
+- # Push all subnets back onto the stack
+- for subnet in reversed(subnets):
+- networks.push(subnet)
++ try:
++ writers[country].write(network)
++ except KeyError:
++ pass
+
+ # Write everything to the filesystem
+ for writer in writers.values():
+@@ -298,33 +222,3 @@ class Exporter(object):
+ )
+
+ return os.path.join(directory, filename)
+-
+-
+-class BufferedStack(object):
+- """
+- This class takes an iterator and when being iterated
+- over it returns objects from that iterator for as long
+- as there are any.
+-
+- It additionally has a function to put an item back on
+- the back so that it will be returned again at the next
+- iteration.
+- """
+- def __init__(self, iterator):
+- self.iterator = iterator
+- self.stack = []
+-
+- def __iter__(self):
+- return self
+-
+- def __next__(self):
+- if self.stack:
+- return self.stack.pop(0)
+-
+- return next(self.iterator)
+-
+- def push(self, elem):
+- """
+- Takes an element and puts it on the stack
+- """
+- self.stack.insert(0, elem)
+--
+2.20.1
+
+From e0b9ff5f38beb0d560b16db881647e5a75127df1 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Sun, 15 Nov 2020 15:02:28 +0000
+Subject: [PATCH 55/70] Move network lists into an own file
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ Makefile.am | 2 +
+ src/libloc.sym | 1 +
+ src/loc/network-list.h | 37 +++++++
+ src/loc/network.h | 20 +---
+ src/network-list.c | 224 +++++++++++++++++++++++++++++++++++++++++
+ src/network.c | 207 +------------------------------------
+ src/python/network.c | 1 +
+ 7 files changed, 269 insertions(+), 223 deletions(-)
+ create mode 100644 src/loc/network-list.h
+ create mode 100644 src/network-list.c
+
+diff --git a/Makefile.am b/Makefile.am
+index a0431a6..f0d8c4c 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -96,6 +96,7 @@ pkginclude_HEADERS = \
+ src/loc/database.h \
+ src/loc/format.h \
+ src/loc/network.h \
++ src/loc/network-list.h \
+ src/loc/private.h \
+ src/loc/stringpool.h \
+ src/loc/resolv.h \
+@@ -110,6 +111,7 @@ src_libloc_la_SOURCES = \
+ src/country.c \
+ src/database.c \
+ src/network.c \
++ src/network-list.c \
+ src/resolv.c \
+ src/stringpool.c \
+ src/writer.c
+diff --git a/src/libloc.sym b/src/libloc.sym
+index 6139db6..453a1be 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -87,6 +87,7 @@ global:
+ loc_network_format_last_address;
+ loc_network_get_asn;
+ loc_network_get_country_code;
++ loc_network_gt;
+ loc_network_has_flag;
+ loc_network_is_subnet;
+ loc_network_is_subnet_of;
+diff --git a/src/loc/network-list.h b/src/loc/network-list.h
+new file mode 100644
+index 0000000..af3b28d
+--- /dev/null
++++ b/src/loc/network-list.h
+@@ -0,0 +1,37 @@
++/*
++ libloc - A library to determine the location of someone on the Internet
++
++ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
++
++ This library is free software; you can redistribute it and/or
++ modify it under the terms of the GNU Lesser General Public
++ License as published by the Free Software Foundation; either
++ version 2.1 of the License, or (at your option) any later version.
++
++ This library is distributed in the hope that it will be useful,
++ but WITHOUT ANY WARRANTY; without even the implied warranty of
++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ Lesser General Public License for more details.
++*/
++
++#ifndef LIBLOC_NETWORK_LIST_H
++#define LIBLOC_NETWORK_LIST_H
++
++struct loc_network_list;
++int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
++struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
++struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
++size_t loc_network_list_size(struct loc_network_list* list);
++int loc_network_list_empty(struct loc_network_list* list);
++void loc_network_list_clear(struct loc_network_list* list);
++void loc_network_list_dump(struct loc_network_list* list);
++struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
++int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
++struct loc_network* loc_network_list_pop(struct loc_network_list* list);
++struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
++int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
++void loc_network_list_sort(struct loc_network_list* list);
++void loc_network_list_reverse(struct loc_network_list* list);
++int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
++
++#endif
+diff --git a/src/loc/network.h b/src/loc/network.h
+index 203e61c..d86b685 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -21,6 +21,7 @@
+
+ #include <loc/libloc.h>
+ #include <loc/format.h>
++#include <loc/network-list.h>
+
+ enum loc_network_flags {
+ LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
+@@ -55,6 +56,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
+ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
+
+ int loc_network_eq(struct loc_network* self, struct loc_network* other);
++int loc_network_gt(struct loc_network* self, struct loc_network* other);
+ int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
+ int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
+ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
+@@ -64,24 +66,6 @@ struct loc_network_list* loc_network_exclude(
+ struct loc_network_list* loc_network_exclude_list(
+ struct loc_network* network, struct loc_network_list* list);
+
+-// List
+-struct loc_network_list;
+-int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
+-struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
+-struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
+-size_t loc_network_list_size(struct loc_network_list* list);
+-int loc_network_list_empty(struct loc_network_list* list);
+-void loc_network_list_clear(struct loc_network_list* list);
+-void loc_network_list_dump(struct loc_network_list* list);
+-struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
+-int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
+-struct loc_network* loc_network_list_pop(struct loc_network_list* list);
+-struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
+-int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
+-void loc_network_list_sort(struct loc_network_list* list);
+-void loc_network_list_reverse(struct loc_network_list* list);
+-int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
+-
+ #ifdef LIBLOC_PRIVATE
+
+ int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
+diff --git a/src/network-list.c b/src/network-list.c
+new file mode 100644
+index 0000000..1f6e80e
+--- /dev/null
++++ b/src/network-list.c
+@@ -0,0 +1,224 @@
++/*
++ libloc - A library to determine the location of someone on the Internet
++
++ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
++
++ This library is free software; you can redistribute it and/or
++ modify it under the terms of the GNU Lesser General Public
++ License as published by the Free Software Foundation; either
++ version 2.1 of the License, or (at your option) any later version.
++
++ This library is distributed in the hope that it will be useful,
++ but WITHOUT ANY WARRANTY; without even the implied warranty of
++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ Lesser General Public License for more details.
++*/
++
++#include <errno.h>
++#include <stdlib.h>
++
++#include <loc/libloc.h>
++#include <loc/network.h>
++#include <loc/private.h>
++
++struct loc_network_list {
++ struct loc_ctx* ctx;
++ int refcount;
++
++ struct loc_network* list[1024];
++ size_t size;
++ size_t max_size;
++};
++
++LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
++ struct loc_network_list** list) {
++ struct loc_network_list* l = calloc(1, sizeof(*l));
++ if (!l)
++ return -ENOMEM;
++
++ l->ctx = loc_ref(ctx);
++ l->refcount = 1;
++
++ // Do not allow this list to grow larger than this
++ l->max_size = 1024;
++
++ DEBUG(l->ctx, "Network list allocated at %p\n", l);
++ *list = l;
++ return 0;
++}
++
++LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
++ list->refcount++;
++
++ return list;
++}
++
++static void loc_network_list_free(struct loc_network_list* list) {
++ DEBUG(list->ctx, "Releasing network list at %p\n", list);
++
++ for (unsigned int i = 0; i < list->size; i++)
++ loc_network_unref(list->list[i]);
++
++ loc_unref(list->ctx);
++ free(list);
++}
++
++LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
++ if (!list)
++ return NULL;
++
++ if (--list->refcount > 0)
++ return list;
++
++ loc_network_list_free(list);
++ return NULL;
++}
++
++LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
++ return list->size;
++}
++
++LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
++ return list->size == 0;
++}
++
++LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
++ for (unsigned int i = 0; i < list->size; i++)
++ loc_network_unref(list->list[i]);
++
++ list->size = 0;
++}
++
++LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
++ struct loc_network* network;
++ char* s;
++
++ for (unsigned int i = 0; i < list->size; i++) {
++ network = list->list[i];
++
++ s = loc_network_str(network);
++
++ INFO(list->ctx, "%s\n", s);
++ free(s);
++ }
++}
++
++LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
++ // Check index
++ if (index >= list->size)
++ return NULL;
++
++ return loc_network_ref(list->list[index]);
++}
++
++LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
++ // Do not add networks that are already on the list
++ if (loc_network_list_contains(list, network))
++ return 0;
++
++ // Check if we have space left
++ if (list->size == list->max_size) {
++ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
++ return -ENOMEM;
++ }
++
++ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
++
++ list->list[list->size++] = loc_network_ref(network);
++
++ return 0;
++}
++
++LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
++ // Return nothing when empty
++ if (loc_network_list_empty(list)) {
++ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
++ return NULL;
++ }
++
++ struct loc_network* network = list->list[--list->size];
++
++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
++
++ return network;
++}
++
++LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
++ // Return nothing when empty
++ if (loc_network_list_empty(list)) {
++ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
++ return NULL;
++ }
++
++ struct loc_network* network = list->list[0];
++
++ // Move all elements to the top of the stack
++ for (unsigned int i = 0; i < --list->size; i++) {
++ list->list[i] = list->list[i+1];
++ }
++
++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
++
++ return network;
++}
++
++LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
++ for (unsigned int i = 0; i < list->size; i++) {
++ if (loc_network_eq(list->list[i], network))
++ return 1;
++ }
++
++ return 0;
++}
++
++static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
++ // Do nothing for invalid indices
++ if (i1 >= list->size || i2 >= list->size)
++ return;
++
++ struct loc_network* network1 = list->list[i1];
++ struct loc_network* network2 = list->list[i2];
++
++ list->list[i1] = network2;
++ list->list[i2] = network1;
++}
++
++LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
++ unsigned int i = 0;
++ unsigned int j = list->size - 1;
++
++ while (i < j) {
++ loc_network_list_swap(list, i++, j--);
++ }
++}
++
++LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
++ unsigned int n = list->size;
++ int swapped;
++
++ do {
++ swapped = 0;
++
++ for (unsigned int i = 1; i < n; i++) {
++ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
++ loc_network_list_swap(list, i-1, i);
++ swapped = 1;
++ }
++ }
++
++ n--;
++ } while (swapped);
++}
++
++LOC_EXPORT int loc_network_list_merge(
++ struct loc_network_list* self, struct loc_network_list* other) {
++ int r;
++
++ for (unsigned int i = 0; i < other->size; i++) {
++ r = loc_network_list_push(self, other->list[i]);
++ if (r)
++ return r;
++ }
++
++ return 0;
++}
+diff --git a/src/network.c b/src/network.c
+index d015579..28ca2df 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -29,6 +29,7 @@
+ #include <loc/compat.h>
+ #include <loc/country.h>
+ #include <loc/network.h>
++#include <loc/network-list.h>
+ #include <loc/private.h>
+
+ struct loc_network {
+@@ -436,7 +437,7 @@ LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* othe
+ return 1;
+ }
+
+-static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
++LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
+ // Families must match
+ if (self->family != other->family)
+ return -1;
+@@ -1106,207 +1107,3 @@ int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
+ struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
+ return loc_network_ref(node->network);
+ }
+-
+-// List
+-
+-struct loc_network_list {
+- struct loc_ctx* ctx;
+- int refcount;
+-
+- struct loc_network* list[1024];
+- size_t size;
+- size_t max_size;
+-};
+-
+-LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
+- struct loc_network_list** list) {
+- struct loc_network_list* l = calloc(1, sizeof(*l));
+- if (!l)
+- return -ENOMEM;
+-
+- l->ctx = loc_ref(ctx);
+- l->refcount = 1;
+-
+- // Do not allow this list to grow larger than this
+- l->max_size = 1024;
+-
+- DEBUG(l->ctx, "Network list allocated at %p\n", l);
+- *list = l;
+- return 0;
+-}
+-
+-LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
+- list->refcount++;
+-
+- return list;
+-}
+-
+-static void loc_network_list_free(struct loc_network_list* list) {
+- DEBUG(list->ctx, "Releasing network list at %p\n", list);
+-
+- for (unsigned int i = 0; i < list->size; i++)
+- loc_network_unref(list->list[i]);
+-
+- loc_unref(list->ctx);
+- free(list);
+-}
+-
+-LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
+- if (!list)
+- return NULL;
+-
+- if (--list->refcount > 0)
+- return list;
+-
+- loc_network_list_free(list);
+- return NULL;
+-}
+-
+-LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
+- return list->size;
+-}
+-
+-LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
+- return list->size == 0;
+-}
+-
+-LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
+- for (unsigned int i = 0; i < list->size; i++)
+- loc_network_unref(list->list[i]);
+-
+- list->size = 0;
+-}
+-
+-LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
+- struct loc_network* network;
+- char* s;
+-
+- for (unsigned int i = 0; i < list->size; i++) {
+- network = list->list[i];
+-
+- s = loc_network_str(network);
+-
+- INFO(list->ctx, "%s\n", s);
+- free(s);
+- }
+-}
+-
+-LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
+- // Check index
+- if (index >= list->size)
+- return NULL;
+-
+- return loc_network_ref(list->list[index]);
+-}
+-
+-LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
+- // Do not add networks that are already on the list
+- if (loc_network_list_contains(list, network))
+- return 0;
+-
+- // Check if we have space left
+- if (list->size == list->max_size) {
+- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
+- return -ENOMEM;
+- }
+-
+- DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
+-
+- list->list[list->size++] = loc_network_ref(network);
+-
+- return 0;
+-}
+-
+-LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
+- // Return nothing when empty
+- if (loc_network_list_empty(list)) {
+- DEBUG(list->ctx, "%p: Popped empty stack\n", list);
+- return NULL;
+- }
+-
+- struct loc_network* network = list->list[--list->size];
+-
+- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
+-
+- return network;
+-}
+-
+-LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
+- // Return nothing when empty
+- if (loc_network_list_empty(list)) {
+- DEBUG(list->ctx, "%p: Popped empty stack\n", list);
+- return NULL;
+- }
+-
+- struct loc_network* network = list->list[0];
+-
+- // Move all elements to the top of the stack
+- for (unsigned int i = 0; i < --list->size; i++) {
+- list->list[i] = list->list[i+1];
+- }
+-
+- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
+-
+- return network;
+-}
+-
+-LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
+- for (unsigned int i = 0; i < list->size; i++) {
+- if (loc_network_eq(list->list[i], network))
+- return 1;
+- }
+-
+- return 0;
+-}
+-
+-static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
+- // Do nothing for invalid indices
+- if (i1 >= list->size || i2 >= list->size)
+- return;
+-
+- struct loc_network* network1 = list->list[i1];
+- struct loc_network* network2 = list->list[i2];
+-
+- list->list[i1] = network2;
+- list->list[i2] = network1;
+-}
+-
+-LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
+- unsigned int i = 0;
+- unsigned int j = list->size - 1;
+-
+- while (i < j) {
+- loc_network_list_swap(list, i++, j--);
+- }
+-}
+-
+-LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
+- unsigned int n = list->size;
+- int swapped;
+-
+- do {
+- swapped = 0;
+-
+- for (unsigned int i = 1; i < n; i++) {
+- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
+- loc_network_list_swap(list, i-1, i);
+- swapped = 1;
+- }
+- }
+-
+- n--;
+- } while (swapped);
+-}
+-
+-LOC_EXPORT int loc_network_list_merge(
+- struct loc_network_list* self, struct loc_network_list* other) {
+- int r;
+-
+- for (unsigned int i = 0; i < other->size; i++) {
+- r = loc_network_list_push(self, other->list[i]);
+- if (r)
+- return r;
+- }
+-
+- return 0;
+-}
+diff --git a/src/python/network.c b/src/python/network.c
+index 11f672b..ed91d65 100644
+--- a/src/python/network.c
++++ b/src/python/network.c
+@@ -20,6 +20,7 @@
+
+ #include <loc/libloc.h>
+ #include <loc/network.h>
++#include <loc/network-list.h>
+
+ #include "locationmodule.h"
+ #include "network.h"
+--
+2.20.1
+
+From e646a8f35ec7eff009414b3fd107c9af5cf39a86 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Mon, 16 Nov 2020 15:13:28 +0000
+Subject: [PATCH 56/70] Implement filtering for multiple countries in the
+ enumerator
+
+This will allow us to speed up the export of the database
+if only a few countries should be returned.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ Makefile.am | 2 +
+ src/country-list.c | 138 +++++++++++++++++++++++++++++++++++++++++
+ src/country.c | 3 +
+ src/database.c | 47 ++++++--------
+ src/libloc.sym | 15 ++++-
+ src/loc/country-list.h | 43 +++++++++++++
+ src/loc/database.h | 5 +-
+ src/python/database.c | 57 ++++++++++++++---
+ 8 files changed, 274 insertions(+), 36 deletions(-)
+ create mode 100644 src/country-list.c
+ create mode 100644 src/loc/country-list.h
+
+diff --git a/Makefile.am b/Makefile.am
+index f0d8c4c..f4ca3c8 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -93,6 +93,7 @@ pkginclude_HEADERS = \
+ src/loc/as.h \
+ src/loc/compat.h \
+ src/loc/country.h \
++ src/loc/country-list.h \
+ src/loc/database.h \
+ src/loc/format.h \
+ src/loc/network.h \
+@@ -109,6 +110,7 @@ src_libloc_la_SOURCES = \
+ src/libloc.c \
+ src/as.c \
+ src/country.c \
++ src/country-list.c \
+ src/database.c \
+ src/network.c \
+ src/network-list.c \
+diff --git a/src/country-list.c b/src/country-list.c
+new file mode 100644
+index 0000000..ae0d71a
+--- /dev/null
++++ b/src/country-list.c
+@@ -0,0 +1,138 @@
++/*
++ libloc - A library to determine the location of someone on the Internet
++
++ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
++
++ This library is free software; you can redistribute it and/or
++ modify it under the terms of the GNU Lesser General Public
++ License as published by the Free Software Foundation; either
++ version 2.1 of the License, or (at your option) any later version.
++
++ This library is distributed in the hope that it will be useful,
++ but WITHOUT ANY WARRANTY; without even the implied warranty of
++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ Lesser General Public License for more details.
++*/
++
++#include <errno.h>
++#include <stdlib.h>
++
++#include <loc/country.h>
++#include <loc/country-list.h>
++#include <loc/private.h>
++
++struct loc_country_list {
++ struct loc_ctx* ctx;
++ int refcount;
++
++ struct loc_country* list[1024];
++ size_t size;
++ size_t max_size;
++};
++
++LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
++ struct loc_country_list** list) {
++ struct loc_country_list* l = calloc(1, sizeof(*l));
++ if (!l)
++ return -ENOMEM;
++
++ l->ctx = loc_ref(ctx);
++ l->refcount = 1;
++
++ // Do not allow this list to grow larger than this
++ l->max_size = 1024;
++
++ DEBUG(l->ctx, "Country list allocated at %p\n", l);
++ *list = l;
++
++ return 0;
++}
++
++LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
++ list->refcount++;
++
++ return list;
++}
++
++static void loc_country_list_free(struct loc_country_list* list) {
++ DEBUG(list->ctx, "Releasing country list at %p\n", list);
++
++ loc_country_list_clear(list);
++
++ loc_unref(list->ctx);
++ free(list);
++}
++
++LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
++ if (!list)
++ return NULL;
++
++ if (--list->refcount > 0)
++ return list;
++
++ loc_country_list_free(list);
++ return NULL;
++}
++
++LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
++ return list->size;
++}
++
++LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
++ return list->size == 0;
++}
++
++LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
++ for (unsigned int i = 0; i < list->size; i++)
++ loc_country_unref(list->list[i]);
++}
++
++LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
++ // Check index
++ if (index >= list->size)
++ return NULL;
++
++ return loc_country_ref(list->list[index]);
++}
++
++LOC_EXPORT int loc_country_list_append(
++ struct loc_country_list* list, struct loc_country* country) {
++ if (loc_country_list_contains(list, country))
++ return 0;
++
++ // Check if we have space left
++ if (list->size == list->max_size) {
++ ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
++ return -ENOMEM;
++ }
++
++ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
++
++ list->list[list->size++] = loc_country_ref(country);
++
++ return 0;
++}
++
++LOC_EXPORT int loc_country_list_contains(
++ struct loc_country_list* list, struct loc_country* country) {
++ for (unsigned int i = 0; i < list->size; i++) {
++ if (loc_country_cmp(country, list->list[i]) == 0)
++ return 1;
++ }
++
++ return 0;
++}
++
++LOC_EXPORT int loc_country_list_contains_code(
++ struct loc_country_list* list, const char* code) {
++ struct loc_country* country;
++
++ int r = loc_country_new(list->ctx, &country, code);
++ if (r)
++ return -1;
++
++ r = loc_country_list_contains(list, country);
++ loc_country_unref(country);
++
++ return r;
++}
+diff --git a/src/country.c b/src/country.c
+index 2ba93e6..7aac0db 100644
+--- a/src/country.c
++++ b/src/country.c
+@@ -34,6 +34,9 @@ struct loc_country {
+ };
+
+ LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
++ if (!loc_country_code_is_valid(country_code))
++ return -EINVAL;
++
+ struct loc_country* c = calloc(1, sizeof(*c));
+ if (!c)
+ return -ENOMEM;
+diff --git a/src/database.c b/src/database.c
+index b9d870f..29823b2 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -40,6 +40,7 @@
+ #include <loc/as.h>
+ #include <loc/compat.h>
+ #include <loc/country.h>
++#include <loc/country-list.h>
+ #include <loc/database.h>
+ #include <loc/format.h>
+ #include <loc/network.h>
+@@ -99,7 +100,7 @@ struct loc_database_enumerator {
+
+ // Search string
+ char* string;
+- char country_code[3];
++ struct loc_country_list* countries;
+ uint32_t asn;
+ enum loc_network_flags flags;
+ int family;
+@@ -1017,33 +1018,20 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
+ return 0;
+ }
+
+-LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
+- // Set empty country code
+- if (!country_code || !*country_code) {
+- *enumerator->country_code = '\0';
+- return 0;
+- }
++LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
++ struct loc_database_enumerator* enumerator) {
++ if (!enumerator->countries)
++ return NULL;
+
+- // Treat A1, A2, A3 as special country codes,
+- // but perform search for flags instead
+- if (strcmp(country_code, "A1") == 0) {
+- return loc_database_enumerator_set_flag(enumerator,
+- LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
+- } else if (strcmp(country_code, "A2") == 0) {
+- return loc_database_enumerator_set_flag(enumerator,
+- LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
+- } else if (strcmp(country_code, "A3") == 0) {
+- return loc_database_enumerator_set_flag(enumerator,
+- LOC_NETWORK_FLAG_ANYCAST);
+- }
++ return loc_country_list_ref(enumerator->countries);
++}
+
+- // Country codes must be two characters
+- if (!loc_country_code_is_valid(country_code))
+- return -EINVAL;
++LOC_EXPORT int loc_database_enumerator_set_countries(
++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
++ if (enumerator->countries)
++ loc_country_list_unref(enumerator->countries);
+
+- for (unsigned int i = 0; i < 3; i++) {
+- enumerator->country_code[i] = country_code[i];
+- }
++ enumerator->countries = loc_country_list_ref(countries);
+
+ return 0;
+ }
+@@ -1129,6 +1117,12 @@ static int loc_database_enumerator_stack_push_node(
+ return 0;
+ }
+
++static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
++ const char* country_code = loc_network_get_country_code(network);
++
++ return loc_country_list_contains_code(countries, country_code);
++}
++
+ static int loc_database_enumerator_filter_network(
+ struct loc_database_enumerator* enumerator, struct loc_network* network) {
+ // Skip if the family does not match
+@@ -1136,8 +1130,7 @@ static int loc_database_enumerator_filter_network(
+ return 1;
+
+ // Skip if the country code does not match
+- if (*enumerator->country_code &&
+- !loc_network_match_country_code(network, enumerator->country_code))
++ if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
+ return 1;
+
+ // Skip if the ASN does not match
+diff --git a/src/libloc.sym b/src/libloc.sym
+index 453a1be..40e9f88 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -49,6 +49,18 @@ global:
+ loc_country_set_name;
+ loc_country_unref;
+
++ # Country List
++ loc_country_list_append;
++ loc_country_list_clear;
++ loc_country_list_contains;
++ loc_country_list_contains_code;
++ loc_country_list_empty;
++ loc_country_list_get;
++ loc_country_list_new;
++ loc_country_list_ref;
++ loc_country_list_size;
++ loc_country_list_unref;
++
+ # Database
+ loc_database_add_as;
+ loc_database_count_as;
+@@ -66,13 +78,14 @@ global:
+ loc_database_verify;
+
+ # Database Enumerator
++ loc_database_enumerator_get_countries;
+ loc_database_enumerator_new;
+ loc_database_enumerator_next_as;
+ loc_database_enumerator_next_country;
+ loc_database_enumerator_next_network;
+ loc_database_enumerator_ref;
+ loc_database_enumerator_set_asn;
+- loc_database_enumerator_set_country_code;
++ loc_database_enumerator_set_countries;
+ loc_database_enumerator_set_family;
+ loc_database_enumerator_set_flag;
+ loc_database_enumerator_set_string;
+diff --git a/src/loc/country-list.h b/src/loc/country-list.h
+new file mode 100644
+index 0000000..a7f818a
+--- /dev/null
++++ b/src/loc/country-list.h
+@@ -0,0 +1,43 @@
++/*
++ libloc - A library to determine the location of someone on the Internet
++
++ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
++
++ This library is free software; you can redistribute it and/or
++ modify it under the terms of the GNU Lesser General Public
++ License as published by the Free Software Foundation; either
++ version 2.1 of the License, or (at your option) any later version.
++
++ This library is distributed in the hope that it will be useful,
++ but WITHOUT ANY WARRANTY; without even the implied warranty of
++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ Lesser General Public License for more details.
++*/
++
++#ifndef LIBLOC_COUNTRY_LIST_H
++#define LIBLOC_COUNTRY_LIST_H
++
++#include <stdlib.h>
++
++#include <loc/libloc.h>
++#include <loc/country.h>
++
++struct loc_country_list;
++
++int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
++struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
++struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
++
++size_t loc_country_list_size(struct loc_country_list* list);
++int loc_country_list_empty(struct loc_country_list* list);
++void loc_country_list_clear(struct loc_country_list* list);
++
++struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
++int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
++
++int loc_country_list_contains(
++ struct loc_country_list* list, struct loc_country* country);
++int loc_country_list_contains_code(
++ struct loc_country_list* list, const char* code);
++
++#endif
+diff --git a/src/loc/database.h b/src/loc/database.h
+index 14eb5ea..246e5c5 100644
+--- a/src/loc/database.h
++++ b/src/loc/database.h
+@@ -25,6 +25,7 @@
+ #include <loc/network.h>
+ #include <loc/as.h>
+ #include <loc/country.h>
++#include <loc/country-list.h>
+
+ struct loc_database;
+ int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
+@@ -66,7 +67,9 @@ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_
+ struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
+
+ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
+-int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
++struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
++int loc_database_enumerator_set_countries(
++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
+ int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
+ int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
+ int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
+diff --git a/src/python/database.c b/src/python/database.c
+index d169547..e6f6f37 100644
+--- a/src/python/database.c
++++ b/src/python/database.c
+@@ -258,14 +258,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
+ }
+
+ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
+- char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
+- const char* country_code = NULL;
++ char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
++ PyObject* country_codes = NULL;
+ unsigned int asn = 0;
+ int flags = 0;
+ int family = 0;
+ int flatten = 0;
+
+- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
++ &PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
+ return NULL;
+
+ struct loc_database_enumerator* enumerator;
+@@ -277,13 +278,55 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
+ }
+
+ // Set country code we are searching for
+- if (country_code) {
+- r = loc_database_enumerator_set_country_code(enumerator, country_code);
+-
++ if (country_codes) {
++ struct loc_country_list* countries;
++ r = loc_country_list_new(loc_ctx, &countries);
+ if (r) {
+- PyErr_SetFromErrno(PyExc_SystemError);
++ PyErr_SetString(PyExc_SystemError, "Could not create country list");
+ return NULL;
+ }
++
++ for (unsigned int i = 0; i < PyList_Size(country_codes); i++) {
++ PyObject* item = PyList_GetItem(country_codes, i);
++
++ if (!PyUnicode_Check(item)) {
++ PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
++ loc_country_list_unref(countries);
++ return NULL;
++ }
++
++ const char* country_code = PyUnicode_AsUTF8(item);
++
++ struct loc_country* country;
++ r = loc_country_new(loc_ctx, &country, country_code);
++ if (r) {
++ if (r == -EINVAL) {
++ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
++ } else {
++ PyErr_SetString(PyExc_SystemError, "Could not create country");
++ }
++
++ loc_country_list_unref(countries);
++ return NULL;
++ }
++
++ // Append it to the list
++ r = loc_country_list_append(countries, country);
++ if (r) {
++ PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
++
++ loc_country_list_unref(countries);
++ loc_country_unref(country);
++ return NULL;
++ }
++
++ loc_country_unref(country);
++ }
++
++ loc_database_enumerator_set_countries(enumerator, countries);
++
++ Py_DECREF(country_codes);
++ loc_country_list_unref(countries);
+ }
+
+ // Set the ASN we are searching for
+--
+2.20.1
+
+From 7af51f8a579c79714992a3e175036fb511139310 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Mon, 16 Nov 2020 15:20:50 +0000
+Subject: [PATCH 57/70] python: Only return country codes we want
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/export.py | 8 +++++++-
+ 1 file changed, 7 insertions(+), 1 deletion(-)
+
+diff --git a/src/python/export.py b/src/python/export.py
+index be4a68e..5e7fe53 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -184,8 +184,14 @@ class Exporter(object):
+
+ writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
+
++ # Filter countries from special country codes
++ country_codes = [
++ country_code for country_code in countries if not country_code in flags.values()
++ ]
++
+ # Get all networks that match the family
+- networks = self.db.search_networks(family=family, flatten=True)
++ networks = self.db.search_networks(family=family,
++ country_codes=country_codes, flatten=True)
+
+ # Walk through all networks
+ for network in networks:
+--
+2.20.1
+
+From bd1dc6bf6fe4ce40bf12e7426e283b31afd274e1 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Mon, 16 Nov 2020 15:25:15 +0000
+Subject: [PATCH 58/70] database: Filter flags in C
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/export.py | 16 +++++++++++-----
+ 1 file changed, 11 insertions(+), 5 deletions(-)
+
+diff --git a/src/python/export.py b/src/python/export.py
+index 5e7fe53..739742f 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -29,7 +29,7 @@ import _location
+ log = logging.getLogger("location.export")
+ log.propagate = 1
+
+-flags = {
++FLAGS = {
+ _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
+ _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
+ _location.NETWORK_FLAG_ANYCAST : "A3",
+@@ -186,12 +186,18 @@ class Exporter(object):
+
+ # Filter countries from special country codes
+ country_codes = [
+- country_code for country_code in countries if not country_code in flags.values()
++ country_code for country_code in countries if not country_code in FLAGS.values()
+ ]
+
++ # Collect flags
++ flags = 0
++ for flag in FLAGS:
++ if FLAGS[flag] in countries:
++ flags |= flag
++
+ # Get all networks that match the family
+ networks = self.db.search_networks(family=family,
+- country_codes=country_codes, flatten=True)
++ country_codes=country_codes, flags=flags, flatten=True)
+
+ # Walk through all networks
+ for network in networks:
+@@ -208,10 +214,10 @@ class Exporter(object):
+ pass
+
+ # Handle flags
+- for flag in flags:
++ for flag in FLAGS:
+ if network.has_flag(flag):
+ # Fetch the "fake" country code
+- country = flags[flag]
++ country = FLAGS[flag]
+
+ try:
+ writers[country].write(network)
+--
+2.20.1
+
+From 84a2f0c2d9cbf8ae4225802c29ccba86561c77ed Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 17 Nov 2020 16:46:48 +0000
+Subject: [PATCH 59/70] as: Add list for easier processing
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ Makefile.am | 2 +
+ src/as-list.c | 138 ++++++++++++++++++++++++++++++++++++++++++
+ src/database.c | 29 +++++++--
+ src/libloc.sym | 15 ++++-
+ src/loc/as-list.h | 41 +++++++++++++
+ src/loc/database.h | 5 +-
+ src/python/database.c | 58 ++++++++++++++++--
+ src/python/export.py | 2 +-
+ 8 files changed, 275 insertions(+), 15 deletions(-)
+ create mode 100644 src/as-list.c
+ create mode 100644 src/loc/as-list.h
+
+diff --git a/Makefile.am b/Makefile.am
+index f4ca3c8..d0cc793 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -91,6 +91,7 @@ EXTRA_DIST += \
+ pkginclude_HEADERS = \
+ src/loc/libloc.h \
+ src/loc/as.h \
++ src/loc/as-list.h \
+ src/loc/compat.h \
+ src/loc/country.h \
+ src/loc/country-list.h \
+@@ -109,6 +110,7 @@ lib_LTLIBRARIES = \
+ src_libloc_la_SOURCES = \
+ src/libloc.c \
+ src/as.c \
++ src/as-list.c \
+ src/country.c \
+ src/country-list.c \
+ src/database.c \
+diff --git a/src/as-list.c b/src/as-list.c
+new file mode 100644
+index 0000000..7c69eb0
+--- /dev/null
++++ b/src/as-list.c
+@@ -0,0 +1,138 @@
++/*
++ libloc - A library to determine the location of someone on the Internet
++
++ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
++
++ This library is free software; you can redistribute it and/or
++ modify it under the terms of the GNU Lesser General Public
++ License as published by the Free Software Foundation; either
++ version 2.1 of the License, or (at your option) any later version.
++
++ This library is distributed in the hope that it will be useful,
++ but WITHOUT ANY WARRANTY; without even the implied warranty of
++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ Lesser General Public License for more details.
++*/
++
++#include <errno.h>
++#include <stdlib.h>
++
++#include <loc/as.h>
++#include <loc/as-list.h>
++#include <loc/private.h>
++
++struct loc_as_list {
++ struct loc_ctx* ctx;
++ int refcount;
++
++ struct loc_as* list[1024];
++ size_t size;
++ size_t max_size;
++};
++
++LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
++ struct loc_as_list** list) {
++ struct loc_as_list* l = calloc(1, sizeof(*l));
++ if (!l)
++ return -ENOMEM;
++
++ l->ctx = loc_ref(ctx);
++ l->refcount = 1;
++
++ // Do not allow this list to grow larger than this
++ l->max_size = 1024;
++
++ DEBUG(l->ctx, "AS list allocated at %p\n", l);
++ *list = l;
++
++ return 0;
++}
++
++LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
++ list->refcount++;
++
++ return list;
++}
++
++static void loc_as_list_free(struct loc_as_list* list) {
++ DEBUG(list->ctx, "Releasing AS list at %p\n", list);
++
++ loc_as_list_clear(list);
++
++ loc_unref(list->ctx);
++ free(list);
++}
++
++LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
++ if (!list)
++ return NULL;
++
++ if (--list->refcount > 0)
++ return list;
++
++ loc_as_list_free(list);
++ return NULL;
++}
++
++LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
++ return list->size;
++}
++
++LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
++ return list->size == 0;
++}
++
++LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
++ for (unsigned int i = 0; i < list->size; i++)
++ loc_as_unref(list->list[i]);
++}
++
++LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
++ // Check index
++ if (index >= list->size)
++ return NULL;
++
++ return loc_as_ref(list->list[index]);
++}
++
++LOC_EXPORT int loc_as_list_append(
++ struct loc_as_list* list, struct loc_as* as) {
++ if (loc_as_list_contains(list, as))
++ return 0;
++
++ // Check if we have space left
++ if (list->size == list->max_size) {
++ ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
++ return -ENOMEM;
++ }
++
++ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
++
++ list->list[list->size++] = loc_as_ref(as);
++
++ return 0;
++}
++
++LOC_EXPORT int loc_as_list_contains(
++ struct loc_as_list* list, struct loc_as* as) {
++ for (unsigned int i = 0; i < list->size; i++) {
++ if (loc_as_cmp(as, list->list[i]) == 0)
++ return 1;
++ }
++
++ return 0;
++}
++
++LOC_EXPORT int loc_as_list_contains_number(
++ struct loc_as_list* list, uint32_t number) {
++ struct loc_as* as;
++
++ int r = loc_as_new(list->ctx, &as, number);
++ if (r)
++ return -1;
++
++ r = loc_as_list_contains(list, as);
++ loc_as_unref(as);
++
++ return r;
++}
+diff --git a/src/database.c b/src/database.c
+index 29823b2..51cb5cd 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -38,6 +38,7 @@
+
+ #include <loc/libloc.h>
+ #include <loc/as.h>
++#include <loc/as-list.h>
+ #include <loc/compat.h>
+ #include <loc/country.h>
+ #include <loc/country-list.h>
+@@ -101,7 +102,7 @@ struct loc_database_enumerator {
+ // Search string
+ char* string;
+ struct loc_country_list* countries;
+- uint32_t asn;
++ struct loc_as_list* asns;
+ enum loc_network_flags flags;
+ int family;
+
+@@ -1036,9 +1037,20 @@ LOC_EXPORT int loc_database_enumerator_set_countries(
+ return 0;
+ }
+
+-LOC_EXPORT int loc_database_enumerator_set_asn(
+- struct loc_database_enumerator* enumerator, unsigned int asn) {
+- enumerator->asn = asn;
++LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
++ struct loc_database_enumerator* enumerator) {
++ if (!enumerator->asns)
++ return NULL;
++
++ return loc_as_list_ref(enumerator->asns);
++}
++
++LOC_EXPORT int loc_database_enumerator_set_asns(
++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
++ if (enumerator->asns)
++ loc_as_list_unref(enumerator->asns);
++
++ enumerator->asns = loc_as_list_ref(asns);
+
+ return 0;
+ }
+@@ -1123,6 +1135,12 @@ static int loc_network_match_countries(struct loc_network* network, struct loc_c
+ return loc_country_list_contains_code(countries, country_code);
+ }
+
++static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
++ uint32_t asn = loc_network_get_asn(network);
++
++ return loc_as_list_contains_number(asns, asn);
++}
++
+ static int loc_database_enumerator_filter_network(
+ struct loc_database_enumerator* enumerator, struct loc_network* network) {
+ // Skip if the family does not match
+@@ -1134,8 +1152,7 @@ static int loc_database_enumerator_filter_network(
+ return 1;
+
+ // Skip if the ASN does not match
+- if (enumerator->asn &&
+- !loc_network_match_asn(network, enumerator->asn))
++ if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
+ return 1;
+
+ // Skip if flags do not match
+diff --git a/src/libloc.sym b/src/libloc.sym
+index 40e9f88..53273cd 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -37,6 +37,18 @@ global:
+ loc_as_set_name;
+ loc_as_unref;
+
++ # AS List
++ loc_as_list_append;
++ loc_as_list_clear;
++ loc_as_list_contains;
++ loc_as_list_contains_number;
++ loc_as_list_empty;
++ loc_as_list_get;
++ loc_as_list_new;
++ loc_as_list_ref;
++ loc_as_list_size;
++ loc_as_list_unref;
++
+ # Country
+ loc_country_cmp;
+ loc_country_code_is_valid;
+@@ -78,13 +90,14 @@ global:
+ loc_database_verify;
+
+ # Database Enumerator
++ loc_database_enumerator_get_asns;
+ loc_database_enumerator_get_countries;
+ loc_database_enumerator_new;
+ loc_database_enumerator_next_as;
+ loc_database_enumerator_next_country;
+ loc_database_enumerator_next_network;
+ loc_database_enumerator_ref;
+- loc_database_enumerator_set_asn;
++ loc_database_enumerator_set_asns;
+ loc_database_enumerator_set_countries;
+ loc_database_enumerator_set_family;
+ loc_database_enumerator_set_flag;
+diff --git a/src/loc/as-list.h b/src/loc/as-list.h
+new file mode 100644
+index 0000000..7b5c4e8
+--- /dev/null
++++ b/src/loc/as-list.h
+@@ -0,0 +1,41 @@
++/*
++ libloc - A library to determine the location of someone on the Internet
++
++ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
++
++ This library is free software; you can redistribute it and/or
++ modify it under the terms of the GNU Lesser General Public
++ License as published by the Free Software Foundation; either
++ version 2.1 of the License, or (at your option) any later version.
++
++ This library is distributed in the hope that it will be useful,
++ but WITHOUT ANY WARRANTY; without even the implied warranty of
++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ Lesser General Public License for more details.
++*/
++
++#ifndef LIBLOC_AS_LIST_H
++#define LIBLOC_AS_LIST_H
++
++#include <loc/as.h>
++#include <loc/libloc.h>
++
++struct loc_as_list;
++
++int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
++struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
++struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
++
++size_t loc_as_list_size(struct loc_as_list* list);
++int loc_as_list_empty(struct loc_as_list* list);
++void loc_as_list_clear(struct loc_as_list* list);
++
++struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
++int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
++
++int loc_as_list_contains(
++ struct loc_as_list* list, struct loc_as* as);
++int loc_as_list_contains_number(
++ struct loc_as_list* list, uint32_t number);
++
++#endif
+diff --git a/src/loc/database.h b/src/loc/database.h
+index 246e5c5..70801f0 100644
+--- a/src/loc/database.h
++++ b/src/loc/database.h
+@@ -70,7 +70,10 @@ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerato
+ struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
+ int loc_database_enumerator_set_countries(
+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
+-int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
++struct loc_as_list* loc_database_enumerator_get_asns(
++ struct loc_database_enumerator* enumerator);
++int loc_database_enumerator_set_asns(
++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
+ int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
+ int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
+ int loc_database_enumerator_next_as(
+diff --git a/src/python/database.c b/src/python/database.c
+index e6f6f37..38a804c 100644
+--- a/src/python/database.c
++++ b/src/python/database.c
+@@ -17,6 +17,8 @@
+ #include <Python.h>
+
+ #include <loc/libloc.h>
++#include <loc/as.h>
++#include <loc/as-list.h>
+ #include <loc/database.h>
+
+ #include "locationmodule.h"
+@@ -258,15 +260,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
+ }
+
+ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
+- char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
++ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
+ PyObject* country_codes = NULL;
+- unsigned int asn = 0;
++ PyObject* asn_list = NULL;
+ int flags = 0;
+ int family = 0;
+ int flatten = 0;
+
+- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
+- &PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
++ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
+ return NULL;
+
+ struct loc_database_enumerator* enumerator;
+@@ -330,13 +332,57 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
+ }
+
+ // Set the ASN we are searching for
+- if (asn) {
+- r = loc_database_enumerator_set_asn(enumerator, asn);
++ if (asn_list) {
++ struct loc_as_list* asns;
++ r = loc_as_list_new(loc_ctx, &asns);
++ if (r) {
++ PyErr_SetString(PyExc_SystemError, "Could not create AS list");
++ return NULL;
++ }
++
++ for (unsigned int i = 0; i < PyList_Size(asn_list); i++) {
++ PyObject* item = PyList_GetItem(asn_list, i);
++
++ if (!PyLong_Check(item)) {
++ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
++
++ loc_as_list_unref(asns);
++ return NULL;
++ }
++
++ unsigned long number = PyLong_AsLong(item);
+
++ struct loc_as* as;
++ r = loc_as_new(loc_ctx, &as, number);
++ if (r) {
++ PyErr_SetString(PyExc_SystemError, "Could not create AS");
++
++ loc_as_list_unref(asns);
++ loc_as_unref(as);
++ return NULL;
++ }
++
++ r = loc_as_list_append(asns, as);
++ if (r) {
++ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
++
++ loc_as_list_unref(asns);
++ loc_as_unref(as);
++ return NULL;
++ }
++
++ loc_as_unref(as);
++ }
++
++ r = loc_database_enumerator_set_asns(enumerator, asns);
+ if (r) {
+ PyErr_SetFromErrno(PyExc_SystemError);
++
++ loc_as_list_unref(asns);
+ return NULL;
+ }
++
++ loc_as_list_unref(asns);
+ }
+
+ // Set the flags we are searching for
+diff --git a/src/python/export.py b/src/python/export.py
+index 739742f..f675eb3 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -197,7 +197,7 @@ class Exporter(object):
+
+ # Get all networks that match the family
+ networks = self.db.search_networks(family=family,
+- country_codes=country_codes, flags=flags, flatten=True)
++ country_codes=country_codes, asns=asns, flags=flags, flatten=True)
+
+ # Walk through all networks
+ for network in networks:
+--
+2.20.1
+
+From 50120b991fc2fa4b7813096de87b42d700faf3e6 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 17 Nov 2020 16:56:43 +0000
+Subject: [PATCH 60/70] database: Simplify network matching code
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/database.c | 16 ++++++++--------
+ 1 file changed, 8 insertions(+), 8 deletions(-)
+
+diff --git a/src/database.c b/src/database.c
+index 51cb5cd..1a354f6 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
+ return 0;
+ }
+
+-static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
+- const char* country_code = loc_network_get_country_code(network);
+-
+- return loc_country_list_contains_code(countries, country_code);
+-}
+-
+ static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
+ uint32_t asn = loc_network_get_asn(network);
+
+@@ -1148,8 +1142,14 @@ static int loc_database_enumerator_filter_network(
+ return 1;
+
+ // Skip if the country code does not match
+- if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
+- return 1;
++ if (enumerator->countries) {
++ if (!loc_country_list_empty(enumerator->countries)) {
++ const char* country_code = loc_network_get_country_code(network);
++
++ if (!loc_country_list_contains_code(enumerator->countries, country_code))
++ return 1;
++ }
++ }
+
+ // Skip if the ASN does not match
+ if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
+--
+2.20.1
+
+From c1a36c943181da5cd2aef589a972d5027e529eb8 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 17 Nov 2020 16:58:55 +0000
+Subject: [PATCH 61/70] database: Simplify AS matching code
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/database.c | 16 ++++++++--------
+ 1 file changed, 8 insertions(+), 8 deletions(-)
+
+diff --git a/src/database.c b/src/database.c
+index 1a354f6..be93e00 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
+ return 0;
+ }
+
+-static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
+- uint32_t asn = loc_network_get_asn(network);
+-
+- return loc_as_list_contains_number(asns, asn);
+-}
+-
+ static int loc_database_enumerator_filter_network(
+ struct loc_database_enumerator* enumerator, struct loc_network* network) {
+ // Skip if the family does not match
+@@ -1152,8 +1146,14 @@ static int loc_database_enumerator_filter_network(
+ }
+
+ // Skip if the ASN does not match
+- if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
+- return 1;
++ if (enumerator->asns) {
++ if (!loc_as_list_empty(enumerator->asns)) {
++ uint32_t asn = loc_network_get_asn(network);
++
++ if (!loc_as_list_contains_number(enumerator->asns, asn))
++ return 1;
++ }
++ }
+
+ // Skip if flags do not match
+ if (enumerator->flags &&
+--
+2.20.1
+
+From d5205091f9cc1ff987e483325d48696459df08d8 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 17 Nov 2020 17:50:17 +0000
+Subject: [PATCH 62/70] countries: Make list grow dynamically
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/country-list.c | 38 ++++++++++++++++++++++++++------------
+ 1 file changed, 26 insertions(+), 12 deletions(-)
+
+diff --git a/src/country-list.c b/src/country-list.c
+index ae0d71a..1ce2d06 100644
+--- a/src/country-list.c
++++ b/src/country-list.c
+@@ -25,11 +25,27 @@ struct loc_country_list {
+ struct loc_ctx* ctx;
+ int refcount;
+
+- struct loc_country* list[1024];
++ struct loc_country** elements;
++ size_t elements_size;
++
+ size_t size;
+- size_t max_size;
+ };
+
++static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
++ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
++ list, size, list->elements_size + size);
++
++ struct loc_country** elements = reallocarray(list->elements,
++ list->elements_size + size, sizeof(*list->elements));
++ if (!elements)
++ return -errno;
++
++ list->elements = elements;
++ list->elements_size += size;
++
++ return 0;
++}
++
+ LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
+ struct loc_country_list** list) {
+ struct loc_country_list* l = calloc(1, sizeof(*l));
+@@ -39,9 +55,6 @@ LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
+ l->ctx = loc_ref(ctx);
+ l->refcount = 1;
+
+- // Do not allow this list to grow larger than this
+- l->max_size = 1024;
+-
+ DEBUG(l->ctx, "Country list allocated at %p\n", l);
+ *list = l;
+
+@@ -84,7 +97,7 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
+
+ LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
+ for (unsigned int i = 0; i < list->size; i++)
+- loc_country_unref(list->list[i]);
++ loc_country_unref(list->elements[i]);
+ }
+
+ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
+@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* lis
+ if (index >= list->size)
+ return NULL;
+
+- return loc_country_ref(list->list[index]);
++ return loc_country_ref(list->elements[index]);
+ }
+
+ LOC_EXPORT int loc_country_list_append(
+@@ -101,14 +114,15 @@ LOC_EXPORT int loc_country_list_append(
+ return 0;
+
+ // Check if we have space left
+- if (list->size == list->max_size) {
+- ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
+- return -ENOMEM;
++ if (list->size >= list->elements_size) {
++ int r = loc_country_list_grow(list, 64);
++ if (r)
++ return r;
+ }
+
+ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
+
+- list->list[list->size++] = loc_country_ref(country);
++ list->elements[list->size++] = loc_country_ref(country);
+
+ return 0;
+ }
+@@ -116,7 +130,7 @@ LOC_EXPORT int loc_country_list_append(
+ LOC_EXPORT int loc_country_list_contains(
+ struct loc_country_list* list, struct loc_country* country) {
+ for (unsigned int i = 0; i < list->size; i++) {
+- if (loc_country_cmp(country, list->list[i]) == 0)
++ if (loc_country_cmp(country, list->elements[i]) == 0)
+ return 1;
+ }
+
+--
+2.20.1
+
+From 3b44e4211371d2103f89ba8f056b15edb7778fac Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 17 Nov 2020 17:55:51 +0000
+Subject: [PATCH 63/70] networks: Make list grow dynamically
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/network-list.c | 60 ++++++++++++++++++++++++++++------------------
+ 1 file changed, 37 insertions(+), 23 deletions(-)
+
+diff --git a/src/network-list.c b/src/network-list.c
+index 1f6e80e..4912c02 100644
+--- a/src/network-list.c
++++ b/src/network-list.c
+@@ -25,11 +25,27 @@ struct loc_network_list {
+ struct loc_ctx* ctx;
+ int refcount;
+
+- struct loc_network* list[1024];
++ struct loc_network** elements;
++ size_t elements_size;
++
+ size_t size;
+- size_t max_size;
+ };
+
++static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
++ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
++ list, size, list->elements_size + size);
++
++ struct loc_network** elements = reallocarray(list->elements,
++ list->elements_size + size, sizeof(*list->elements));
++ if (!elements)
++ return -errno;
++
++ list->elements = elements;
++ list->elements_size += size;
++
++ return 0;
++}
++
+ LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
+ struct loc_network_list** list) {
+ struct loc_network_list* l = calloc(1, sizeof(*l));
+@@ -39,9 +55,6 @@ LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
+ l->ctx = loc_ref(ctx);
+ l->refcount = 1;
+
+- // Do not allow this list to grow larger than this
+- l->max_size = 1024;
+-
+ DEBUG(l->ctx, "Network list allocated at %p\n", l);
+ *list = l;
+ return 0;
+@@ -57,7 +70,7 @@ static void loc_network_list_free(struct loc_network_list* list) {
+ DEBUG(list->ctx, "Releasing network list at %p\n", list);
+
+ for (unsigned int i = 0; i < list->size; i++)
+- loc_network_unref(list->list[i]);
++ loc_network_unref(list->elements[i]);
+
+ loc_unref(list->ctx);
+ free(list);
+@@ -84,7 +97,7 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
+
+ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
+ for (unsigned int i = 0; i < list->size; i++)
+- loc_network_unref(list->list[i]);
++ loc_network_unref(list->elements[i]);
+
+ list->size = 0;
+ }
+@@ -94,7 +107,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
+ char* s;
+
+ for (unsigned int i = 0; i < list->size; i++) {
+- network = list->list[i];
++ network = list->elements[i];
+
+ s = loc_network_str(network);
+
+@@ -108,7 +121,7 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
+ if (index >= list->size)
+ return NULL;
+
+- return loc_network_ref(list->list[index]);
++ return loc_network_ref(list->elements[index]);
+ }
+
+ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
+@@ -117,14 +130,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
+ return 0;
+
+ // Check if we have space left
+- if (list->size == list->max_size) {
+- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
+- return -ENOMEM;
++ if (list->size >= list->elements_size) {
++ int r = loc_network_list_grow(list, 64);
++ if (r)
++ return r;
+ }
+
+ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
+
+- list->list[list->size++] = loc_network_ref(network);
++ list->elements[list->size++] = loc_network_ref(network);
+
+ return 0;
+ }
+@@ -136,7 +150,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
+ return NULL;
+ }
+
+- struct loc_network* network = list->list[--list->size];
++ struct loc_network* network = list->elements[--list->size];
+
+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
+
+@@ -150,11 +164,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
+ return NULL;
+ }
+
+- struct loc_network* network = list->list[0];
++ struct loc_network* network = list->elements[0];
+
+ // Move all elements to the top of the stack
+ for (unsigned int i = 0; i < --list->size; i++) {
+- list->list[i] = list->list[i+1];
++ list->elements[i] = list->elements[i+1];
+ }
+
+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
+@@ -164,7 +178,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
+
+ LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
+ for (unsigned int i = 0; i < list->size; i++) {
+- if (loc_network_eq(list->list[i], network))
++ if (loc_network_eq(list->elements[i], network))
+ return 1;
+ }
+
+@@ -176,11 +190,11 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
+ if (i1 >= list->size || i2 >= list->size)
+ return;
+
+- struct loc_network* network1 = list->list[i1];
+- struct loc_network* network2 = list->list[i2];
++ struct loc_network* network1 = list->elements[i1];
++ struct loc_network* network2 = list->elements[i2];
+
+- list->list[i1] = network2;
+- list->list[i2] = network1;
++ list->elements[i1] = network2;
++ list->elements[i2] = network1;
+ }
+
+ LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
+@@ -200,7 +214,7 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
+ swapped = 0;
+
+ for (unsigned int i = 1; i < n; i++) {
+- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
++ if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) {
+ loc_network_list_swap(list, i-1, i);
+ swapped = 1;
+ }
+@@ -215,7 +229,7 @@ LOC_EXPORT int loc_network_list_merge(
+ int r;
+
+ for (unsigned int i = 0; i < other->size; i++) {
+- r = loc_network_list_push(self, other->list[i]);
++ r = loc_network_list_push(self, other->elements[i]);
+ if (r)
+ return r;
+ }
+--
+2.20.1
+
+From 1a415f8c555f4fe9a68eb2a897c4a1fc0d33db25 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 17 Nov 2020 17:57:55 +0000
+Subject: [PATCH 64/70] as: Make lists grow dynamically
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/as-list.c | 38 ++++++++++++++++++++++++++------------
+ 1 file changed, 26 insertions(+), 12 deletions(-)
+
+diff --git a/src/as-list.c b/src/as-list.c
+index 7c69eb0..17de23e 100644
+--- a/src/as-list.c
++++ b/src/as-list.c
+@@ -25,11 +25,27 @@ struct loc_as_list {
+ struct loc_ctx* ctx;
+ int refcount;
+
+- struct loc_as* list[1024];
++ struct loc_as** elements;
++ size_t elements_size;
++
+ size_t size;
+- size_t max_size;
+ };
+
++static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
++ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
++ list, size, list->elements_size + size);
++
++ struct loc_as** elements = reallocarray(list->elements,
++ list->elements_size + size, sizeof(*list->elements));
++ if (!elements)
++ return -errno;
++
++ list->elements = elements;
++ list->elements_size += size;
++
++ return 0;
++}
++
+ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
+ struct loc_as_list** list) {
+ struct loc_as_list* l = calloc(1, sizeof(*l));
+@@ -39,9 +55,6 @@ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
+ l->ctx = loc_ref(ctx);
+ l->refcount = 1;
+
+- // Do not allow this list to grow larger than this
+- l->max_size = 1024;
+-
+ DEBUG(l->ctx, "AS list allocated at %p\n", l);
+ *list = l;
+
+@@ -84,7 +97,7 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
+
+ LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
+ for (unsigned int i = 0; i < list->size; i++)
+- loc_as_unref(list->list[i]);
++ loc_as_unref(list->elements[i]);
+ }
+
+ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
+@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index
+ if (index >= list->size)
+ return NULL;
+
+- return loc_as_ref(list->list[index]);
++ return loc_as_ref(list->elements[index]);
+ }
+
+ LOC_EXPORT int loc_as_list_append(
+@@ -101,14 +114,15 @@ LOC_EXPORT int loc_as_list_append(
+ return 0;
+
+ // Check if we have space left
+- if (list->size == list->max_size) {
+- ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
+- return -ENOMEM;
++ if (list->size >= list->elements_size) {
++ int r = loc_as_list_grow(list, 64);
++ if (r)
++ return r;
+ }
+
+ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
+
+- list->list[list->size++] = loc_as_ref(as);
++ list->elements[list->size++] = loc_as_ref(as);
+
+ return 0;
+ }
+@@ -116,7 +130,7 @@ LOC_EXPORT int loc_as_list_append(
+ LOC_EXPORT int loc_as_list_contains(
+ struct loc_as_list* list, struct loc_as* as) {
+ for (unsigned int i = 0; i < list->size; i++) {
+- if (loc_as_cmp(as, list->list[i]) == 0)
++ if (loc_as_cmp(as, list->elements[i]) == 0)
+ return 1;
+ }
+
+--
+2.20.1
+
+From e6592434ee7836507c1f436ec3b0db3bc81a81b9 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 17 Nov 2020 18:13:49 +0000
+Subject: [PATCH 65/70] export: Change back to use Network objects
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/export.py | 4 +---
+ 1 file changed, 1 insertion(+), 3 deletions(-)
+
+diff --git a/src/python/export.py b/src/python/export.py
+index f675eb3..67e437f 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -144,9 +144,7 @@ class XTGeoIPOutputWriter(OutputWriter):
+ def _write_network(self, network):
+ for address in (network.first_address, network.last_address):
+ # Convert this into a string of bits
+- bytes = socket.inet_pton(
+- socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
+- )
++ bytes = socket.inet_pton(network.family, address)
+
+ self.f.write(bytes)
+
+--
+2.20.1
+
+From 248f5e0419f2349253b8ea96e477c15649fe2173 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 17 Nov 2020 18:14:15 +0000
+Subject: [PATCH 66/70] Actually clear all lists
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/as-list.c | 8 ++++++++
+ src/country-list.c | 8 ++++++++
+ src/network-list.c | 6 ++++++
+ 3 files changed, 22 insertions(+)
+
+diff --git a/src/as-list.c b/src/as-list.c
+index 17de23e..76620c7 100644
+--- a/src/as-list.c
++++ b/src/as-list.c
+@@ -96,8 +96,16 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
+ }
+
+ LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
++ if (!list->elements)
++ return;
++
+ for (unsigned int i = 0; i < list->size; i++)
+ loc_as_unref(list->elements[i]);
++
++ free(list->elements);
++ list->elements_size = 0;
++
++ list->size = 0;
+ }
+
+ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
+diff --git a/src/country-list.c b/src/country-list.c
+index 1ce2d06..1c49c47 100644
+--- a/src/country-list.c
++++ b/src/country-list.c
+@@ -96,8 +96,16 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
+ }
+
+ LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
++ if (!list->elements)
++ return;
++
+ for (unsigned int i = 0; i < list->size; i++)
+ loc_country_unref(list->elements[i]);
++
++ free(list->elements);
++ list->elements_size = 0;
++
++ list->size = 0;
+ }
+
+ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
+diff --git a/src/network-list.c b/src/network-list.c
+index 4912c02..9cb4547 100644
+--- a/src/network-list.c
++++ b/src/network-list.c
+@@ -96,9 +96,15 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
+ }
+
+ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
++ if (!list->elements)
++ return;
++
+ for (unsigned int i = 0; i < list->size; i++)
+ loc_network_unref(list->elements[i]);
+
++ free(list->elements);
++ list->elements_size = 0;
++
+ list->size = 0;
+ }
+
+--
+2.20.1
+
+From c98ebf8aae2aa141193db52cd9429b1ded5b09c4 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 17 Nov 2020 18:34:51 +0000
+Subject: [PATCH 67/70] database: Do not clean up python list
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/database.c | 9 +++++++--
+ 1 file changed, 7 insertions(+), 2 deletions(-)
+
+diff --git a/src/python/database.c b/src/python/database.c
+index 38a804c..ed22275 100644
+--- a/src/python/database.c
++++ b/src/python/database.c
+@@ -325,9 +325,14 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
+ loc_country_unref(country);
+ }
+
+- loc_database_enumerator_set_countries(enumerator, countries);
++ r = loc_database_enumerator_set_countries(enumerator, countries);
++ if (r) {
++ PyErr_SetFromErrno(PyExc_SystemError);
++
++ loc_as_list_unref(countries);
++ return NULL;
++ }
+
+- Py_DECREF(country_codes);
+ loc_country_list_unref(countries);
+ }
+
+--
+2.20.1
+
+From 5470d06cb59027f4e04b6d576763dbf7f1093fde Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Tue, 17 Nov 2020 19:01:04 +0000
+Subject: [PATCH 68/70] database: Free filter lists in enumerator
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/database.c | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/src/database.c b/src/database.c
+index be93e00..ca35fe1 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -950,6 +950,12 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera
+ if (enumerator->string)
+ free(enumerator->string);
+
++ if (enumerator->countries)
++ loc_country_list_unref(enumerator->countries);
++
++ if (enumerator->asns)
++ loc_as_list_unref(enumerator->asns);
++
+ // Free network search
+ free(enumerator->networks_visited);
+
+--
+2.20.1
+
+From e0e96878d3df51c4a265d51d088005dedf9335e3 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 18 Nov 2020 13:18:52 +0000
+Subject: [PATCH 69/70] database: Add debug output to filtering
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/database.c | 29 ++++++++++++++++-------------
+ 1 file changed, 16 insertions(+), 13 deletions(-)
+
+diff --git a/src/database.c b/src/database.c
+index ca35fe1..83dd752 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -1138,33 +1138,36 @@ static int loc_database_enumerator_stack_push_node(
+ static int loc_database_enumerator_filter_network(
+ struct loc_database_enumerator* enumerator, struct loc_network* network) {
+ // Skip if the family does not match
+- if (enumerator->family && loc_network_address_family(network) != enumerator->family)
++ if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
++ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
+ return 1;
++ }
+
+ // Skip if the country code does not match
+- if (enumerator->countries) {
+- if (!loc_country_list_empty(enumerator->countries)) {
+- const char* country_code = loc_network_get_country_code(network);
++ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
++ const char* country_code = loc_network_get_country_code(network);
+
+- if (!loc_country_list_contains_code(enumerator->countries, country_code))
+- return 1;
++ if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
++ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
++ return 1;
+ }
+ }
+
+ // Skip if the ASN does not match
+- if (enumerator->asns) {
+- if (!loc_as_list_empty(enumerator->asns)) {
+- uint32_t asn = loc_network_get_asn(network);
++ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
++ uint32_t asn = loc_network_get_asn(network);
+
+- if (!loc_as_list_contains_number(enumerator->asns, asn))
+- return 1;
++ if (!loc_as_list_contains_number(enumerator->asns, asn)) {
++ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
++ return 1;
+ }
+ }
+
+ // Skip if flags do not match
+- if (enumerator->flags &&
+- !loc_network_match_flag(network, enumerator->flags))
++ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
++ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
+ return 1;
++ }
+
+ // Do not filter
+ return 0;
+--
+2.20.1
+
+From bce0c9295ff8ff9488f24babe01ce851228d0b1e Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 18 Nov 2020 13:19:04 +0000
+Subject: [PATCH 70/70] export: Remove filtering for flags
+
+The filter is an AND filter and if we set the flags from
+the special country codes, we won't get back much.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/export.py | 8 +-------
+ 1 file changed, 1 insertion(+), 7 deletions(-)
+
+diff --git a/src/python/export.py b/src/python/export.py
+index 67e437f..4219957 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -187,15 +187,9 @@ class Exporter(object):
+ country_code for country_code in countries if not country_code in FLAGS.values()
+ ]
+
+- # Collect flags
+- flags = 0
+- for flag in FLAGS:
+- if FLAGS[flag] in countries:
+- flags |= flag
+-
+ # Get all networks that match the family
+ networks = self.db.search_networks(family=family,
+- country_codes=country_codes, asns=asns, flags=flags, flatten=True)
++ country_codes=country_codes, asns=asns, flatten=True)
+
+ # Walk through all networks
+ for network in networks:
+--
+2.20.1
+
projects / ipfire-2.x.git / commitdiff
