From: Michael Tremer Date: Wed, 18 Nov 2020 13:30:15 +0000 (+0000) Subject: libloc: Import recent patches from upstream X-Git-Tag: v2.25-core155~384^2~47 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=05db64d0ea36e3f4be816d5818ce755d7e02b9d1;p=ipfire-2.x.git libloc: Import recent patches from upstream Signed-off-by: Michael Tremer --- diff --git a/config/rootfiles/common/libloc b/config/rootfiles/common/libloc index b120520748..e8776e56ef 100644 --- a/config/rootfiles/common/libloc +++ b/config/rootfiles/common/libloc @@ -1,12 +1,15 @@ usr/bin/location #usr/bin/location-importer #usr/include/libloc +#usr/include/libloc/as-list.h #usr/include/libloc/as.h #usr/include/libloc/compat.h +#usr/include/libloc/country-list.h #usr/include/libloc/country.h #usr/include/libloc/database.h #usr/include/libloc/format.h #usr/include/libloc/libloc.h +#usr/include/libloc/network-list.h #usr/include/libloc/network.h #usr/include/libloc/private.h #usr/include/libloc/resolv.h diff --git a/lfs/libloc b/lfs/libloc index da53e51494..65270eb24b 100644 --- a/lfs/libloc +++ b/lfs/libloc @@ -78,6 +78,9 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) @$(PREBUILD) @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xvf $(DIR_DL)/$(DL_FILE) + # Import changes from upstream + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.4-upstream.patch + # Add patch for i586 to disable strong stack protector. ifeq "$(BUILD_ARCH)" "i586" cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.3-perl-i586-regular-stack-protector.patch diff --git a/src/patches/libloc-0.9.4-upstream.patch b/src/patches/libloc-0.9.4-upstream.patch new file mode 100644 index 0000000000..28d569d93d --- /dev/null +++ b/src/patches/libloc-0.9.4-upstream.patch @@ -0,0 +1,7629 @@ +From ee6ea3986dc80183157f67275dc9f28231b5d5b2 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 24 Sep 2020 10:17:58 +0000 +Subject: [PATCH 01/70] Revert "importer: Purge any redundant entries" + +This reverts commit c2cc55d5a6875c3838f060032eaed89dcfb92ef6. + +The query stalls the database and therefore the automatic +scripts are no longer able to generate a new version of the +database. + +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 22 +--------------------- + 1 file changed, 1 insertion(+), 21 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 1467923..e3a07a0 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -374,27 +374,7 @@ class CLI(object): + INSERT INTO autnums(number, name) + SELECT _autnums.number, _organizations.name FROM _autnums + JOIN _organizations ON _autnums.organization = _organizations.handle +- ON CONFLICT (number) DO UPDATE SET name = excluded.name +- """) +- +- self.db.execute(""" +- --- Purge any redundant entries +- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP +- AS +- SELECT network FROM networks candidates +- WHERE EXISTS ( +- SELECT FROM networks +- WHERE +- networks.network << candidates.network +- AND +- networks.country = candidates.country +- ); +- +- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network); +- +- DELETE FROM networks WHERE EXISTS ( +- SELECT FROM _garbage WHERE networks.network = _garbage.network +- ); ++ ON CONFLICT (number) DO UPDATE SET name = excluded.name; + """) + + # Download all extended sources +-- +2.20.1 + +From 92f6abf4e272672bb0a71cfe991261b95ebe2fef Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 24 Sep 2020 10:18:58 +0000 +Subject: [PATCH 02/70] Revert "importer: Import raw sources for inetnum's + again" + +This reverts commit 64e95fa903edec8b4e4e59830b395e2e4a411853. + +Signed-off-by: Michael Tremer +--- + src/python/importer.py | 14 ++++---- + src/python/location-importer.in | 63 --------------------------------- + 2 files changed, 7 insertions(+), 70 deletions(-) + +diff --git a/src/python/importer.py b/src/python/importer.py +index f19db4b..de20f37 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( + "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", + + # Asia Pacific Network Information Centre +- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", +- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", ++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", ++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", + "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", +@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( + # XXX ??? + + # Réseaux IP Européens +- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", +- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", ++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", ++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", +@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( + + EXTENDED_SOURCES = ( + # African Network Information Centre +- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", ++ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", + + # Asia Pacific Network Information Centre +- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", ++ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", + + # American Registry for Internet Numbers + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", +@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( + "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", + + # Réseaux IP Européens +- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", ++ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", + ) + + class Downloader(object): +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index e3a07a0..77952f2 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -393,10 +393,6 @@ class CLI(object): + if line.startswith("aut-num:"): + return self._parse_autnum_block(block) + +- # inetnum +- if line.startswith("inet6num:") or line.startswith("inetnum:"): +- return self._parse_inetnum_block(block) +- + # organisation + elif line.startswith("organisation:"): + return self._parse_org_block(block) +@@ -426,65 +422,6 @@ class CLI(object): + autnum.get("asn"), autnum.get("org"), + ) + +- def _parse_inetnum_block(self, block): +- logging.debug("Parsing inetnum block:") +- +- inetnum = {} +- for line in block: +- logging.debug(line) +- +- # Split line +- key, val = split_line(line) +- +- if key == "inetnum": +- start_address, delim, end_address = val.partition("-") +- +- # Strip any excess space +- start_address, end_address = start_address.rstrip(), end_address.strip() +- +- # Convert to IP address +- try: +- start_address = ipaddress.ip_address(start_address) +- end_address = ipaddress.ip_address(end_address) +- except ValueError: +- logging.warning("Could not parse line: %s" % line) +- return +- +- # Set prefix to default +- prefix = 32 +- +- # Count number of addresses in this subnet +- num_addresses = int(end_address) - int(start_address) +- if num_addresses: +- prefix -= math.log(num_addresses, 2) +- +- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) +- +- elif key == "inet6num": +- inetnum[key] = val +- +- elif key == "country": +- if val == "UNITED STATES": +- val = "US" +- +- inetnum[key] = val.upper() +- +- # Skip empty objects +- if not inetnum: +- return +- +- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) +- +- # Bail out in case we have processed a non-public IP network +- if network.is_private: +- logging.warning("Skipping non-globally routable network: %s" % network) +- return +- +- self.db.execute("INSERT INTO networks(network, country) \ +- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", +- "%s" % network, inetnum.get("country"), +- ) +- + def _parse_org_block(self, block): + org = {} + for line in block: +-- +2.20.1 + +From f532841e9197ce2f40aad8c086d786b2cb783a54 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Mon, 12 Oct 2020 20:53:31 +0000 +Subject: [PATCH 03/70] Revert "Revert "importer: Import raw sources for + inetnum's again"" + +This reverts commit 92f6abf4e272672bb0a71cfe991261b95ebe2fef. + +Signed-off-by: Michael Tremer +--- + src/python/importer.py | 14 ++++---- + src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++ + 2 files changed, 70 insertions(+), 7 deletions(-) + +diff --git a/src/python/importer.py b/src/python/importer.py +index de20f37..f19db4b 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( + "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", + + # Asia Pacific Network Information Centre +- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", +- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", ++ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", ++ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", + "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", +@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( + # XXX ??? + + # Réseaux IP Européens +- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", +- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", ++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", ++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", +@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( + + EXTENDED_SOURCES = ( + # African Network Information Centre +- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", ++ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", + + # Asia Pacific Network Information Centre +- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", ++ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", + + # American Registry for Internet Numbers + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", +@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( + "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", + + # Réseaux IP Européens +- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", ++ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", + ) + + class Downloader(object): +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 77952f2..e3a07a0 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -393,6 +393,10 @@ class CLI(object): + if line.startswith("aut-num:"): + return self._parse_autnum_block(block) + ++ # inetnum ++ if line.startswith("inet6num:") or line.startswith("inetnum:"): ++ return self._parse_inetnum_block(block) ++ + # organisation + elif line.startswith("organisation:"): + return self._parse_org_block(block) +@@ -422,6 +426,65 @@ class CLI(object): + autnum.get("asn"), autnum.get("org"), + ) + ++ def _parse_inetnum_block(self, block): ++ logging.debug("Parsing inetnum block:") ++ ++ inetnum = {} ++ for line in block: ++ logging.debug(line) ++ ++ # Split line ++ key, val = split_line(line) ++ ++ if key == "inetnum": ++ start_address, delim, end_address = val.partition("-") ++ ++ # Strip any excess space ++ start_address, end_address = start_address.rstrip(), end_address.strip() ++ ++ # Convert to IP address ++ try: ++ start_address = ipaddress.ip_address(start_address) ++ end_address = ipaddress.ip_address(end_address) ++ except ValueError: ++ logging.warning("Could not parse line: %s" % line) ++ return ++ ++ # Set prefix to default ++ prefix = 32 ++ ++ # Count number of addresses in this subnet ++ num_addresses = int(end_address) - int(start_address) ++ if num_addresses: ++ prefix -= math.log(num_addresses, 2) ++ ++ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) ++ ++ elif key == "inet6num": ++ inetnum[key] = val ++ ++ elif key == "country": ++ if val == "UNITED STATES": ++ val = "US" ++ ++ inetnum[key] = val.upper() ++ ++ # Skip empty objects ++ if not inetnum: ++ return ++ ++ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) ++ ++ # Bail out in case we have processed a non-public IP network ++ if network.is_private: ++ logging.warning("Skipping non-globally routable network: %s" % network) ++ return ++ ++ self.db.execute("INSERT INTO networks(network, country) \ ++ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", ++ "%s" % network, inetnum.get("country"), ++ ) ++ + def _parse_org_block(self, block): + org = {} + for line in block: +-- +2.20.1 + +From a36bc686865fc87ea386fd90b389338bdcb80cbc Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Mon, 12 Oct 2020 20:53:32 +0000 +Subject: [PATCH 04/70] location-importer.in: only import relevant data from + AFRINIC, APNIC and RIPE +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +In contrast to ARIN and LACNIC, we are able to process more detailled +feeds from those RIRs, avoiding storage of obviously unnecessary data. + +Thanks to various SQL optimisations, doing so now takes less time than +the first version of this did. + +Signed-off-by: Michael Tremer +Signed-off-by: Peter Müller +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++- + 1 file changed, 87 insertions(+), 2 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index e3a07a0..093f325 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -165,6 +165,7 @@ class CLI(object): + -- networks + CREATE TABLE IF NOT EXISTS networks(network inet, country text); + CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); ++ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); + CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); + + -- overrides +@@ -363,6 +364,16 @@ class CLI(object): + CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) + ON COMMIT DROP; + CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); ++ ++ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) ++ ON COMMIT DROP; ++ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); ++ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); ++ """) ++ ++ # Remove all previously imported content ++ self.db.execute(""" ++ TRUNCATE TABLE networks; + """) + + for source in location.importer.WHOIS_SOURCES: +@@ -370,6 +381,67 @@ class CLI(object): + for block in f: + self._parse_block(block) + ++ # Process all parsed networks from every RIR we happen to have access to, ++ # insert the largest network chunks into the networks table immediately... ++ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") ++ ++ for family in (row.family for row in families): ++ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) ++ ++ self.db.execute("INSERT INTO networks(network, country) \ ++ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) ++ ++ # ... determine any other prefixes for this network family, ... ++ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ ++ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) ++ ++ # ... and insert networks with this prefix in case they provide additional ++ # information (i. e. subnet of a larger chunk with a different country) ++ for prefix in (row.prefix for row in prefixes): ++ self.db.execute(""" ++ WITH candidates AS ( ++ SELECT ++ _rirdata.network, ++ _rirdata.country ++ FROM ++ _rirdata ++ WHERE ++ family(_rirdata.network) = %s ++ AND ++ masklen(_rirdata.network) = %s ++ ), ++ filtered AS ( ++ SELECT ++ DISTINCT ON (c.network) ++ c.network, ++ c.country, ++ masklen(networks.network), ++ networks.country AS parent_country ++ FROM ++ candidates c ++ LEFT JOIN ++ networks ++ ON ++ c.network << networks.network ++ ORDER BY ++ c.network, ++ masklen(networks.network) DESC NULLS LAST ++ ) ++ INSERT INTO ++ networks(network, country) ++ SELECT ++ network, ++ country ++ FROM ++ filtered ++ WHERE ++ parent_country IS NULL ++ OR ++ country <> parent_country ++ ON CONFLICT DO NOTHING""", ++ family, prefix, ++ ) ++ + self.db.execute(""" + INSERT INTO autnums(number, name) + SELECT _autnums.number, _organizations.name FROM _autnums +@@ -470,17 +542,30 @@ class CLI(object): + inetnum[key] = val.upper() + + # Skip empty objects +- if not inetnum: ++ if not inetnum or not "country" in inetnum: + return + + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) + ++ # Bail out in case we have processed a network covering the entire IP range, which ++ # is necessary to work around faulty (?) IPv6 network processing ++ if network.prefixlen == 0: ++ logging.warning("Skipping network covering the entire IP adress range: %s" % network) ++ return ++ ++ # Bail out in case we have processed a network whose prefix length indicates it is ++ # not globally routable (we have decided not to process them at the moment, as they ++ # significantly enlarge our database without providing very helpful additional information) ++ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): ++ logging.info("Skipping network too small to be publicly announced: %s" % network) ++ return ++ + # Bail out in case we have processed a non-public IP network + if network.is_private: + logging.warning("Skipping non-globally routable network: %s" % network) + return + +- self.db.execute("INSERT INTO networks(network, country) \ ++ self.db.execute("INSERT INTO _rirdata(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", + "%s" % network, inetnum.get("country"), + ) +-- +2.20.1 + +From 2373de384f10f5573bbd7570f5522545df70c0e3 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Fri, 16 Oct 2020 12:24:58 +0000 +Subject: [PATCH 05/70] location-importer: Include all overridden networks + +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 093f325..d249a35 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -240,6 +240,8 @@ class CLI(object): + SELECT network FROM announcements + UNION + SELECT network FROM networks ++ UNION ++ SELECT network FROM network_overrides + ORDER BY network + ) + +-- +2.20.1 + +From 13f67f285856e8eabfeff2daf1be3aeaa36a82cc Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Fri, 16 Oct 2020 12:26:38 +0000 +Subject: [PATCH 06/70] Revert "location-importer.in: only import relevant data + from AFRINIC, APNIC and RIPE" + +This reverts commit a36bc686865fc87ea386fd90b389338bdcb80cbc. + +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 89 +-------------------------------- + 1 file changed, 2 insertions(+), 87 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index d249a35..b220eaf 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -165,7 +165,6 @@ class CLI(object): + -- networks + CREATE TABLE IF NOT EXISTS networks(network inet, country text); + CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); +- CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); + CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); + + -- overrides +@@ -366,16 +365,6 @@ class CLI(object): + CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) + ON COMMIT DROP; + CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); +- +- CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) +- ON COMMIT DROP; +- CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); +- CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); +- """) +- +- # Remove all previously imported content +- self.db.execute(""" +- TRUNCATE TABLE networks; + """) + + for source in location.importer.WHOIS_SOURCES: +@@ -383,67 +372,6 @@ class CLI(object): + for block in f: + self._parse_block(block) + +- # Process all parsed networks from every RIR we happen to have access to, +- # insert the largest network chunks into the networks table immediately... +- families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") +- +- for family in (row.family for row in families): +- smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) +- +- self.db.execute("INSERT INTO networks(network, country) \ +- SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) +- +- # ... determine any other prefixes for this network family, ... +- prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ +- WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) +- +- # ... and insert networks with this prefix in case they provide additional +- # information (i. e. subnet of a larger chunk with a different country) +- for prefix in (row.prefix for row in prefixes): +- self.db.execute(""" +- WITH candidates AS ( +- SELECT +- _rirdata.network, +- _rirdata.country +- FROM +- _rirdata +- WHERE +- family(_rirdata.network) = %s +- AND +- masklen(_rirdata.network) = %s +- ), +- filtered AS ( +- SELECT +- DISTINCT ON (c.network) +- c.network, +- c.country, +- masklen(networks.network), +- networks.country AS parent_country +- FROM +- candidates c +- LEFT JOIN +- networks +- ON +- c.network << networks.network +- ORDER BY +- c.network, +- masklen(networks.network) DESC NULLS LAST +- ) +- INSERT INTO +- networks(network, country) +- SELECT +- network, +- country +- FROM +- filtered +- WHERE +- parent_country IS NULL +- OR +- country <> parent_country +- ON CONFLICT DO NOTHING""", +- family, prefix, +- ) +- + self.db.execute(""" + INSERT INTO autnums(number, name) + SELECT _autnums.number, _organizations.name FROM _autnums +@@ -544,30 +472,17 @@ class CLI(object): + inetnum[key] = val.upper() + + # Skip empty objects +- if not inetnum or not "country" in inetnum: ++ if not inetnum: + return + + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) + +- # Bail out in case we have processed a network covering the entire IP range, which +- # is necessary to work around faulty (?) IPv6 network processing +- if network.prefixlen == 0: +- logging.warning("Skipping network covering the entire IP adress range: %s" % network) +- return +- +- # Bail out in case we have processed a network whose prefix length indicates it is +- # not globally routable (we have decided not to process them at the moment, as they +- # significantly enlarge our database without providing very helpful additional information) +- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): +- logging.info("Skipping network too small to be publicly announced: %s" % network) +- return +- + # Bail out in case we have processed a non-public IP network + if network.is_private: + logging.warning("Skipping non-globally routable network: %s" % network) + return + +- self.db.execute("INSERT INTO _rirdata(network, country) \ ++ self.db.execute("INSERT INTO networks(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", + "%s" % network, inetnum.get("country"), + ) +-- +2.20.1 + +From 44341478233115b26bb27fdb24da5b0a1eedb173 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Fri, 16 Oct 2020 12:26:43 +0000 +Subject: [PATCH 07/70] Revert "Revert "Revert "importer: Import raw sources + for inetnum's again""" + +This reverts commit f532841e9197ce2f40aad8c086d786b2cb783a54. + +Signed-off-by: Michael Tremer +--- + src/python/importer.py | 14 ++++---- + src/python/location-importer.in | 63 --------------------------------- + 2 files changed, 7 insertions(+), 70 deletions(-) + +diff --git a/src/python/importer.py b/src/python/importer.py +index f19db4b..de20f37 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( + "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", + + # Asia Pacific Network Information Centre +- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", +- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", ++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", ++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", + "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", +@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( + # XXX ??? + + # Réseaux IP Européens +- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", +- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", ++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", ++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", +@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( + + EXTENDED_SOURCES = ( + # African Network Information Centre +- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", ++ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", + + # Asia Pacific Network Information Centre +- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", ++ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", + + # American Registry for Internet Numbers + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", +@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( + "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", + + # Réseaux IP Européens +- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", ++ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", + ) + + class Downloader(object): +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index b220eaf..e87d378 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -395,10 +395,6 @@ class CLI(object): + if line.startswith("aut-num:"): + return self._parse_autnum_block(block) + +- # inetnum +- if line.startswith("inet6num:") or line.startswith("inetnum:"): +- return self._parse_inetnum_block(block) +- + # organisation + elif line.startswith("organisation:"): + return self._parse_org_block(block) +@@ -428,65 +424,6 @@ class CLI(object): + autnum.get("asn"), autnum.get("org"), + ) + +- def _parse_inetnum_block(self, block): +- logging.debug("Parsing inetnum block:") +- +- inetnum = {} +- for line in block: +- logging.debug(line) +- +- # Split line +- key, val = split_line(line) +- +- if key == "inetnum": +- start_address, delim, end_address = val.partition("-") +- +- # Strip any excess space +- start_address, end_address = start_address.rstrip(), end_address.strip() +- +- # Convert to IP address +- try: +- start_address = ipaddress.ip_address(start_address) +- end_address = ipaddress.ip_address(end_address) +- except ValueError: +- logging.warning("Could not parse line: %s" % line) +- return +- +- # Set prefix to default +- prefix = 32 +- +- # Count number of addresses in this subnet +- num_addresses = int(end_address) - int(start_address) +- if num_addresses: +- prefix -= math.log(num_addresses, 2) +- +- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) +- +- elif key == "inet6num": +- inetnum[key] = val +- +- elif key == "country": +- if val == "UNITED STATES": +- val = "US" +- +- inetnum[key] = val.upper() +- +- # Skip empty objects +- if not inetnum: +- return +- +- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) +- +- # Bail out in case we have processed a non-public IP network +- if network.is_private: +- logging.warning("Skipping non-globally routable network: %s" % network) +- return +- +- self.db.execute("INSERT INTO networks(network, country) \ +- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", +- "%s" % network, inetnum.get("country"), +- ) +- + def _parse_org_block(self, block): + org = {} + for line in block: +-- +2.20.1 + +From a7d3a7a0565a0e09d3442e5829a0f30f016993b9 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 20 Oct 2020 20:44:43 +0000 +Subject: [PATCH 08/70] as: Fix dereferencing NULL pointer when setting AS name + +Reported-by: Gisle Vanem +Signed-off-by: Michael Tremer +--- + src/as.c | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/src/as.c b/src/as.c +index e1fbb01..8421ac8 100644 +--- a/src/as.c ++++ b/src/as.c +@@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) { + } + + LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) { +- as->name = strdup(name); ++ if (as->name) ++ free(as->name); ++ ++ if (name) ++ as->name = strdup(name); ++ else ++ as->name = NULL; + + return 0; + } +-- +2.20.1 + +From ddb326ad38a7c7202315dd2c6f938313db04ee22 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 21 Oct 2020 09:18:08 +0000 +Subject: [PATCH 09/70] as: Do not attempt to match name when it wasn't set + +Signed-off-by: Michael Tremer +--- + src/as.c | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/src/as.c b/src/as.c +index 8421ac8..757bf3d 100644 +--- a/src/as.c ++++ b/src/as.c +@@ -145,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) { + if (!string) + return 1; + ++ // Cannot match anything when name is not set ++ if (!as->name) ++ return 1; ++ + // Search if string is in name + if (strcasestr(as->name, string) != NULL) + return 1; +-- +2.20.1 + +From d226ad2d97cbcd42ce807d9308569b1b9c5d4e2f Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 21 Oct 2020 09:28:39 +0000 +Subject: [PATCH 10/70] writer: Free array with pointer to ASes, too + +Signed-off-by: Michael Tremer +--- + src/writer.c | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/src/writer.c b/src/writer.c +index 5939cff..160650f 100644 +--- a/src/writer.c ++++ b/src/writer.c +@@ -147,8 +147,11 @@ static void loc_writer_free(struct loc_writer* writer) { + EVP_PKEY_free(writer->private_key2); + + // Unref all AS +- for (unsigned int i = 0; i < writer->as_count; i++) { +- loc_as_unref(writer->as[i]); ++ if (writer->as) { ++ for (unsigned int i = 0; i < writer->as_count; i++) { ++ loc_as_unref(writer->as[i]); ++ } ++ free(writer->as); + } + + // Release network tree +-- +2.20.1 + +From d89a7d62772048ae1bd18d03f69df46b7e1a3d3c Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 21 Oct 2020 09:31:29 +0000 +Subject: [PATCH 11/70] writer: Free countries when the writer is being + destroyed + +Signed-off-by: Michael Tremer +--- + src/writer.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/src/writer.c b/src/writer.c +index 160650f..2f09b56 100644 +--- a/src/writer.c ++++ b/src/writer.c +@@ -154,6 +154,14 @@ static void loc_writer_free(struct loc_writer* writer) { + free(writer->as); + } + ++ // Unref all countries ++ if (writer->countries) { ++ for (unsigned int i = 0; i < writer->countries_count; i++) { ++ loc_country_unref(writer->countries[i]); ++ } ++ free(writer->countries); ++ } ++ + // Release network tree + if (writer->networks) + loc_network_tree_unref(writer->networks); +-- +2.20.1 + +From 0f1aedbc68e3945770c93e0ebd83eed0f555d6f0 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 21 Oct 2020 13:19:44 +0000 +Subject: [PATCH 12/70] tests: Try adding an invalid network + +Signed-off-by: Michael Tremer +--- + src/test-network.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/src/test-network.c b/src/test-network.c +index d38f13d..e908b57 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -160,6 +160,14 @@ int main(int argc, char** argv) { + // Set ASN + loc_network_set_asn(network4, 1024); + ++ // Try adding an invalid network ++ struct loc_network* network; ++ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32"); ++ if (err != -EINVAL) { ++ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); ++ exit(EXIT_FAILURE); ++ } ++ + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +-- +2.20.1 + +From 13ad6e695f9ffc7847b3afe3e9cbcea8fb3a443f Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 21 Oct 2020 13:36:35 +0000 +Subject: [PATCH 13/70] networks: Improve parsing IP addresses + +loc_network_new_from_string() seem to have had some unexpected +behaviour for invalid inputs. + +The function has been tidied up slightly and returns as soon as +some invalid input was detected. + +Signed-off-by: Michael Tremer +--- + src/network.c | 52 +++++++++++++++++++++++++++++++--------------- + src/test-network.c | 7 +++++++ + 2 files changed, 42 insertions(+), 17 deletions(-) + +diff --git a/src/network.c b/src/network.c +index 366caa2..c112a41 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -160,9 +160,10 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network + LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network, + const char* address_string) { + struct in6_addr first_address; +- unsigned int prefix = 0; + char* prefix_string; +- int r = 1; ++ int r = -EINVAL; ++ ++ DEBUG(ctx, "Attempting to parse network %s\n", address_string); + + // Make a copy of the string to work on it + char* buffer = strdup(address_string); +@@ -171,29 +172,46 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo + // Split address and prefix + address_string = strsep(&prefix_string, "/"); + +- // Did we find a prefix? +- if (prefix_string) { +- // Convert prefix to integer +- prefix = strtol(prefix_string, NULL, 10); ++ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string); + +- if (prefix) { +- // Parse the address +- r = loc_parse_address(ctx, address_string, &first_address); ++ // We need to have a prefix ++ if (!prefix_string) { ++ DEBUG(ctx, "No prefix set\n"); ++ goto FAIL; ++ } + +- // Map the prefix to IPv6 if needed +- if (IN6_IS_ADDR_V4MAPPED(&first_address)) +- prefix += 96; +- } ++ // Convert prefix to integer ++ unsigned int prefix = strtol(prefix_string, NULL, 10); ++ ++ // End if the prefix was invalid ++ if (!prefix) { ++ DEBUG(ctx, "The prefix is zero or not a number\n"); ++ goto FAIL; ++ } ++ ++ // Parse the address ++ r = loc_parse_address(ctx, address_string, &first_address); ++ if (r) { ++ DEBUG(ctx, "The address could not be parsed\n"); ++ goto FAIL; + } + ++ // Map the prefix to IPv6 if needed ++ if (IN6_IS_ADDR_V4MAPPED(&first_address)) ++ prefix += 96; ++ ++FAIL: + // Free temporary buffer + free(buffer); + +- if (r == 0) { +- r = loc_network_new(ctx, network, &first_address, prefix); +- } ++ // Exit if the parsing was unsuccessful ++ if (r) ++ return r; ++ ++ DEBUG(ctx, "GOT HERE\n"); + +- return r; ++ // Create a new network ++ return loc_network_new(ctx, network, &first_address, prefix); + } + + LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) { +diff --git a/src/test-network.c b/src/test-network.c +index e908b57..85eca00 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -168,6 +168,13 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++ // Try adding a single address ++ err = loc_writer_add_network(writer, &network, "2001:db8::"); ++ if (err != -EINVAL) { ++ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); ++ exit(EXIT_FAILURE); ++ } ++ + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +-- +2.20.1 + +From 6a467e9345bb5a3d37911c9aaac30019eaa4492b Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 21 Oct 2020 13:43:21 +0000 +Subject: [PATCH 14/70] networks: Test if we can add localhost (IPv6) + +Signed-off-by: Michael Tremer +--- + src/test-network.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/src/test-network.c b/src/test-network.c +index 85eca00..8c7e898 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -175,6 +175,13 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++ // Try adding localhost ++ err = loc_writer_add_network(writer, &network, "::1/128"); ++ if (err != -EINVAL) { ++ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +-- +2.20.1 + +From fc1190aa11e3ff3d2dbf5f4d408c298e7916f46f Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 21 Oct 2020 13:44:50 +0000 +Subject: [PATCH 15/70] networks: Remove accidentially committed debug line + +Signed-off-by: Michael Tremer +--- + src/network.c | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/src/network.c b/src/network.c +index c112a41..be88d75 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -208,8 +208,6 @@ FAIL: + if (r) + return r; + +- DEBUG(ctx, "GOT HERE\n"); +- + // Create a new network + return loc_network_new(ctx, network, &first_address, prefix); + } +-- +2.20.1 + +From a1707d8983898b6878cdd5c68744bcc444e278ed Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 21 Oct 2020 13:53:36 +0000 +Subject: [PATCH 16/70] importer: Add search index to announcements table + +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index e87d378..d0fe5a6 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -152,6 +152,7 @@ class CLI(object): + last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); + CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network); + CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network)); ++ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops); + + -- autnums + CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); +-- +2.20.1 + +From 991baf530d47adb2ed7a15b65dc4565d07fa6d07 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 21 Oct 2020 13:54:45 +0000 +Subject: [PATCH 17/70] importer: Add search index to network_overrides table + +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index d0fe5a6..fe21d73 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -189,6 +189,8 @@ class CLI(object): + ); + CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network + ON network_overrides(network); ++ CREATE INDEX IF NOT EXISTS network_overrides_search ++ ON network_overrides USING GIST(network inet_ops); + """) + + return db +-- +2.20.1 + +From bbea93a74651df10e2ffdbd09eb434dc6a0471bc Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 21 Oct 2020 16:01:57 +0000 +Subject: [PATCH 18/70] importer: Restructure SQL query to be executed in + parallel + +There are no functional changes, this just runs quicker now. + +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 67 ++++++++++++++++++--------------- + 1 file changed, 37 insertions(+), 30 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index fe21d73..3c1e5e2 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -237,34 +237,24 @@ class CLI(object): + + # Select all known networks + rows = self.db.query(""" +- -- Get a (sorted) list of all known networks +- WITH known_networks AS ( +- SELECT network FROM announcements +- UNION +- SELECT network FROM networks +- UNION +- SELECT network FROM network_overrides +- ORDER BY network +- ) +- + -- Return a list of those networks enriched with all + -- other information that we store in the database + SELECT +- DISTINCT ON (known_networks.network) +- known_networks.network AS network, +- announcements.autnum AS autnum, ++ DISTINCT ON (network) ++ network, ++ autnum, + + -- Country + COALESCE( + ( + SELECT country FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT country FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + networks.country + ) AS country, +@@ -273,50 +263,67 @@ class CLI(object): + COALESCE( + ( + SELECT is_anonymous_proxy FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anonymous_proxy FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_anonymous_proxy, + COALESCE( + ( + SELECT is_satellite_provider FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_satellite_provider FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_satellite_provider, + COALESCE( + ( + SELECT is_anycast FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anycast FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE +- ) AS is_anycast, +- +- -- Must be part of returned values for ORDER BY clause +- masklen(announcements.network) AS sort_a, +- masklen(networks.network) AS sort_b +- FROM known_networks +- LEFT JOIN announcements ON known_networks.network <<= announcements.network +- LEFT JOIN networks ON known_networks.network <<= networks.network +- ORDER BY known_networks.network, sort_a DESC, sort_b DESC ++ ) AS is_anycast ++ FROM ( ++ SELECT ++ known_networks.network AS network, ++ announcements.autnum AS autnum, ++ networks.country AS country, ++ ++ -- Must be part of returned values for ORDER BY clause ++ masklen(announcements.network) AS sort_a, ++ masklen(networks.network) AS sort_b ++ FROM ( ++ SELECT network FROM announcements ++ UNION ALL ++ SELECT network FROM networks ++ UNION ALL ++ SELECT network FROM network_overrides ++ ) known_networks ++ LEFT JOIN ++ announcements ON known_networks.network <<= announcements.network ++ LEFT JOIN ++ networks ON known_networks.network <<= networks.network ++ ORDER BY ++ known_networks.network, ++ sort_a DESC, ++ sort_b DESC ++ ) networks + """) + + for row in rows: +-- +2.20.1 + +From 26ab419b68d166f932db1f97c38cb9d793d04187 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 22 Oct 2020 12:24:34 +0000 +Subject: [PATCH 19/70] network: Allow adding single IP addresses and + automatically add the prefix + +Signed-off-by: Michael Tremer +--- + src/network.c | 33 +++++++++++++++------------------ + src/test-network.c | 4 ++-- + 2 files changed, 17 insertions(+), 20 deletions(-) + +diff --git a/src/network.c b/src/network.c +index be88d75..d7b1645 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -161,6 +161,7 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo + const char* address_string) { + struct in6_addr first_address; + char* prefix_string; ++ unsigned int prefix = 128; + int r = -EINVAL; + + DEBUG(ctx, "Attempting to parse network %s\n", address_string); +@@ -174,21 +175,6 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo + + DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string); + +- // We need to have a prefix +- if (!prefix_string) { +- DEBUG(ctx, "No prefix set\n"); +- goto FAIL; +- } +- +- // Convert prefix to integer +- unsigned int prefix = strtol(prefix_string, NULL, 10); +- +- // End if the prefix was invalid +- if (!prefix) { +- DEBUG(ctx, "The prefix is zero or not a number\n"); +- goto FAIL; +- } +- + // Parse the address + r = loc_parse_address(ctx, address_string, &first_address); + if (r) { +@@ -196,9 +182,20 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo + goto FAIL; + } + +- // Map the prefix to IPv6 if needed +- if (IN6_IS_ADDR_V4MAPPED(&first_address)) +- prefix += 96; ++ // If a prefix was given, we will try to parse it ++ if (prefix_string) { ++ // Convert prefix to integer ++ prefix = strtol(prefix_string, NULL, 10); ++ ++ if (!prefix) { ++ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string); ++ goto FAIL; ++ } ++ ++ // Map the prefix to IPv6 if needed ++ if (IN6_IS_ADDR_V4MAPPED(&first_address)) ++ prefix += 96; ++ } + + FAIL: + // Free temporary buffer +diff --git a/src/test-network.c b/src/test-network.c +index 8c7e898..b6776b4 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -170,8 +170,8 @@ int main(int argc, char** argv) { + + // Try adding a single address + err = loc_writer_add_network(writer, &network, "2001:db8::"); +- if (err != -EINVAL) { +- fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); ++ if (err) { ++ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err); + exit(EXIT_FAILURE); + } + +-- +2.20.1 + +From aadac4c569e921be1d28dd3b2377ac7f3732213e Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Wed, 21 Oct 2020 14:47:36 +0000 +Subject: [PATCH 20/70] Revert "Revert "Revert "Revert "importer: Import raw + sources for inetnum's again"""" +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This reverts commit 44341478233115b26bb27fdb24da5b0a1eedb173. + +Signed-off-by: Peter Müller +Signed-off-by: Michael Tremer +--- + src/python/importer.py | 14 ++++---- + src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++ + 2 files changed, 70 insertions(+), 7 deletions(-) + +diff --git a/src/python/importer.py b/src/python/importer.py +index de20f37..f19db4b 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( + "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", + + # Asia Pacific Network Information Centre +- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", +- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", ++ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", ++ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", + "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", +@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( + # XXX ??? + + # Réseaux IP Européens +- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", +- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", ++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", ++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", +@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( + + EXTENDED_SOURCES = ( + # African Network Information Centre +- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", ++ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", + + # Asia Pacific Network Information Centre +- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", ++ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", + + # American Registry for Internet Numbers + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", +@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( + "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", + + # Réseaux IP Européens +- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", ++ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", + ) + + class Downloader(object): +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 3c1e5e2..e8a4fc5 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -405,6 +405,10 @@ class CLI(object): + if line.startswith("aut-num:"): + return self._parse_autnum_block(block) + ++ # inetnum ++ if line.startswith("inet6num:") or line.startswith("inetnum:"): ++ return self._parse_inetnum_block(block) ++ + # organisation + elif line.startswith("organisation:"): + return self._parse_org_block(block) +@@ -434,6 +438,65 @@ class CLI(object): + autnum.get("asn"), autnum.get("org"), + ) + ++ def _parse_inetnum_block(self, block): ++ logging.debug("Parsing inetnum block:") ++ ++ inetnum = {} ++ for line in block: ++ logging.debug(line) ++ ++ # Split line ++ key, val = split_line(line) ++ ++ if key == "inetnum": ++ start_address, delim, end_address = val.partition("-") ++ ++ # Strip any excess space ++ start_address, end_address = start_address.rstrip(), end_address.strip() ++ ++ # Convert to IP address ++ try: ++ start_address = ipaddress.ip_address(start_address) ++ end_address = ipaddress.ip_address(end_address) ++ except ValueError: ++ logging.warning("Could not parse line: %s" % line) ++ return ++ ++ # Set prefix to default ++ prefix = 32 ++ ++ # Count number of addresses in this subnet ++ num_addresses = int(end_address) - int(start_address) ++ if num_addresses: ++ prefix -= math.log(num_addresses, 2) ++ ++ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) ++ ++ elif key == "inet6num": ++ inetnum[key] = val ++ ++ elif key == "country": ++ if val == "UNITED STATES": ++ val = "US" ++ ++ inetnum[key] = val.upper() ++ ++ # Skip empty objects ++ if not inetnum: ++ return ++ ++ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) ++ ++ # Bail out in case we have processed a non-public IP network ++ if network.is_private: ++ logging.warning("Skipping non-globally routable network: %s" % network) ++ return ++ ++ self.db.execute("INSERT INTO networks(network, country) \ ++ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", ++ "%s" % network, inetnum.get("country"), ++ ) ++ + def _parse_org_block(self, block): + org = {} + for line in block: +-- +2.20.1 + +From 002deb6b42ac0b3624c07e3352cebd72dc0685a2 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Wed, 21 Oct 2020 14:47:37 +0000 +Subject: [PATCH 21/70] Revert "Revert "location-importer.in: only import + relevant data from AFRINIC, APNIC and RIPE"" +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This reverts commit 13f67f285856e8eabfeff2daf1be3aeaa36a82cc. + +Signed-off-by: Peter Müller +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++- + 1 file changed, 87 insertions(+), 2 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index e8a4fc5..5656c41 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -166,6 +166,7 @@ class CLI(object): + -- networks + CREATE TABLE IF NOT EXISTS networks(network inet, country text); + CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); ++ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); + CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); + + -- overrides +@@ -375,6 +376,16 @@ class CLI(object): + CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) + ON COMMIT DROP; + CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); ++ ++ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) ++ ON COMMIT DROP; ++ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); ++ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); ++ """) ++ ++ # Remove all previously imported content ++ self.db.execute(""" ++ TRUNCATE TABLE networks; + """) + + for source in location.importer.WHOIS_SOURCES: +@@ -382,6 +393,67 @@ class CLI(object): + for block in f: + self._parse_block(block) + ++ # Process all parsed networks from every RIR we happen to have access to, ++ # insert the largest network chunks into the networks table immediately... ++ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") ++ ++ for family in (row.family for row in families): ++ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) ++ ++ self.db.execute("INSERT INTO networks(network, country) \ ++ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) ++ ++ # ... determine any other prefixes for this network family, ... ++ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ ++ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) ++ ++ # ... and insert networks with this prefix in case they provide additional ++ # information (i. e. subnet of a larger chunk with a different country) ++ for prefix in (row.prefix for row in prefixes): ++ self.db.execute(""" ++ WITH candidates AS ( ++ SELECT ++ _rirdata.network, ++ _rirdata.country ++ FROM ++ _rirdata ++ WHERE ++ family(_rirdata.network) = %s ++ AND ++ masklen(_rirdata.network) = %s ++ ), ++ filtered AS ( ++ SELECT ++ DISTINCT ON (c.network) ++ c.network, ++ c.country, ++ masklen(networks.network), ++ networks.country AS parent_country ++ FROM ++ candidates c ++ LEFT JOIN ++ networks ++ ON ++ c.network << networks.network ++ ORDER BY ++ c.network, ++ masklen(networks.network) DESC NULLS LAST ++ ) ++ INSERT INTO ++ networks(network, country) ++ SELECT ++ network, ++ country ++ FROM ++ filtered ++ WHERE ++ parent_country IS NULL ++ OR ++ country <> parent_country ++ ON CONFLICT DO NOTHING""", ++ family, prefix, ++ ) ++ + self.db.execute(""" + INSERT INTO autnums(number, name) + SELECT _autnums.number, _organizations.name FROM _autnums +@@ -482,17 +554,30 @@ class CLI(object): + inetnum[key] = val.upper() + + # Skip empty objects +- if not inetnum: ++ if not inetnum or not "country" in inetnum: + return + + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) + ++ # Bail out in case we have processed a network covering the entire IP range, which ++ # is necessary to work around faulty (?) IPv6 network processing ++ if network.prefixlen == 0: ++ logging.warning("Skipping network covering the entire IP adress range: %s" % network) ++ return ++ ++ # Bail out in case we have processed a network whose prefix length indicates it is ++ # not globally routable (we have decided not to process them at the moment, as they ++ # significantly enlarge our database without providing very helpful additional information) ++ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): ++ logging.info("Skipping network too small to be publicly announced: %s" % network) ++ return ++ + # Bail out in case we have processed a non-public IP network + if network.is_private: + logging.warning("Skipping non-globally routable network: %s" % network) + return + +- self.db.execute("INSERT INTO networks(network, country) \ ++ self.db.execute("INSERT INTO _rirdata(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", + "%s" % network, inetnum.get("country"), + ) +-- +2.20.1 + +From 28c73fa3f4257e0a41e52af8a9643da414a6cb6f Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Wed, 21 Oct 2020 14:47:38 +0000 +Subject: [PATCH 22/70] export.py: fix exporting IP networks for crappy + xt_geoip module +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +In contrast to the location database itself, the xt_geoip module +consumes a list of IP networks for each country, and returns after the +first match. + +We therefore need to... + +(a) sort IP networks by their size, allow as precise matches as possible +(b) export _any_ IP networks - including inverted subnets - to prevent + undefined overlaps +(c) do the entire thing as fast as possible, consuming as less disk + space as possible, which is why we can't just iterate over all /24 + chunks + +Partially fixes: #12499 + +Signed-off-by: Michael Tremer +Signed-off-by: Peter Müller +Signed-off-by: Michael Tremer +--- + src/python/export.py | 69 ++++++++++++++++++++++++++++++++++---------- + 1 file changed, 54 insertions(+), 15 deletions(-) + +diff --git a/src/python/export.py b/src/python/export.py +index d15c6f0..5eaf43f 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -39,8 +39,8 @@ class OutputWriter(object): + suffix = "networks" + mode = "w" + +- def __init__(self, f, prefix=None, flatten=True): +- self.f, self.prefix, self.flatten = f, prefix, flatten ++ def __init__(self, db, f, prefix=None, flatten=True): ++ self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten + + # The previously written network + self._last_network = None +@@ -49,13 +49,13 @@ class OutputWriter(object): + self._write_header() + + @classmethod +- def open(cls, filename, **kwargs): ++ def open(cls, db, filename, **kwargs): + """ + Convenience function to open a file + """ + f = open(filename, cls.mode) + +- return cls(f, **kwargs) ++ return cls(db, f, **kwargs) + + def __repr__(self): + return "<%s f=%s>" % (self.__class__.__name__, self.f) +@@ -87,13 +87,31 @@ class OutputWriter(object): + def _write_network(self, network): + self.f.write("%s\n" % network) + +- def write(self, network): ++ def write(self, network, subnets): + if self.flatten and self._flatten(network): + log.debug("Skipping writing network %s" % network) + return + +- # Write the network to file +- self._write_network(network) ++ # Write the network when it has no subnets ++ if not subnets: ++ network = ipaddress.ip_network("%s" % network) ++ return self._write_network(network) ++ ++ # Collect all matching subnets ++ matching_subnets = [] ++ ++ for subnet in sorted(subnets): ++ # Try to find the subnet in the database ++ n = self.db.lookup("%s" % subnet.network_address) ++ ++ # No entry or matching country means those IP addresses belong here ++ if not n or n.country_code == network.country_code: ++ matching_subnets.append(subnet) ++ ++ # Write all networks as compact as possible ++ for network in ipaddress.collapse_addresses(matching_subnets): ++ log.debug("Writing %s to database" % network) ++ self._write_network(network) + + def finish(self): + """ +@@ -143,10 +161,10 @@ class XTGeoIPOutputWriter(OutputWriter): + mode = "wb" + + def _write_network(self, network): +- for address in (network.first_address, network.last_address): ++ for address in (network.network_address, network.broadcast_address): + # Convert this into a string of bits + bytes = socket.inet_pton( +- network.family, address, ++ socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, + ) + + self.f.write(bytes) +@@ -175,7 +193,7 @@ class Exporter(object): + directory, prefix=country_code, suffix=self.writer.suffix, family=family, + ) + +- writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code) ++ writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code) + + # Create writers for ASNs + for asn in asns: +@@ -183,22 +201,43 @@ class Exporter(object): + directory, "AS%s" % asn, suffix=self.writer.suffix, family=family, + ) + +- writers[asn] = self.writer.open(filename, prefix="AS%s" % asn) ++ writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) + + # Get all networks that match the family + networks = self.db.search_networks(family=family) + ++ # Materialise the generator into a list (uses quite some memory) ++ networks = list(networks) ++ + # Walk through all networks +- for network in networks: ++ for i, network in enumerate(networks): ++ _network = ipaddress.ip_network("%s" % network) ++ ++ # Search for all subnets ++ subnets = set() ++ ++ while i < len(networks): ++ subnet = networks[i+1] ++ ++ if subnet.is_subnet_of(network): ++ _subnet = ipaddress.ip_network("%s" % subnet) ++ ++ subnets.add(_subnet) ++ subnets.update(_network.address_exclude(_subnet)) ++ ++ i += 1 ++ else: ++ break ++ + # Write matching countries + try: +- writers[network.country_code].write(network) ++ writers[network.country_code].write(network, subnets) + except KeyError: + pass + + # Write matching ASNs + try: +- writers[network.asn].write(network) ++ writers[network.asn].write(network, subnets) + except KeyError: + pass + +@@ -209,7 +248,7 @@ class Exporter(object): + country = flags[flag] + + try: +- writers[country].write(network) ++ writers[country].write(network, subnets) + except KeyError: + pass + +-- +2.20.1 + +From bd341642fc6bbcc050e9b4ec5124585c83cab84d Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Wed, 21 Oct 2020 14:47:39 +0000 +Subject: [PATCH 23/70] location-importer.in: filter bogus IP networks for both + Whois and extended sources +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Sanity checks for parsed networks have been put into a separate function +to avoid boilerplate code for extended sources. This makes the location +database less vulnerable to garbage written into RIR databases on +purpose or by chance. + +Fixes: #12500 + +Signed-off-by: Peter Müller +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 83 ++++++++++++++++++++++++++------- + 1 file changed, 67 insertions(+), 16 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 5656c41..f24d357 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -469,6 +469,69 @@ class CLI(object): + for line in f: + self._parse_line(line) + ++ def _check_parsed_network(self, network): ++ """ ++ Assistive function to detect and subsequently sort out parsed ++ networks from RIR data (both Whois and so-called "extended sources"), ++ which are or have... ++ ++ (a) not globally routable (RFC 1918 space, et al.) ++ (b) covering a too large chunk of the IP address space (prefix length ++ is < 7 for IPv4 networks, and < 10 for IPv6) ++ (c) "0.0.0.0" or "::" as a network address ++ (d) are too small for being publicly announced (we have decided not to ++ process them at the moment, as they significantly enlarge our ++ database without providing very helpful additional information) ++ ++ This unfortunately is necessary due to brain-dead clutter across ++ various RIR databases, causing mismatches and eventually disruptions. ++ ++ We will return False in case a network is not suitable for adding ++ it to our database, and True otherwise. ++ """ ++ ++ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)): ++ return False ++ ++ if not network.is_global: ++ logging.warning("Skipping non-globally routable network: %s" % network) ++ return False ++ ++ if network.version == 4: ++ if network.prefixlen < 7: ++ logging.warning("Skipping too big IP chunk: %s" % network) ++ return False ++ ++ if network.prefixlen > 24: ++ logging.info("Skipping network too small to be publicly announced: %s" % network) ++ return False ++ ++ if str(network.network_address) == "0.0.0.0": ++ logging.warning("Skipping network based on 0.0.0.0: %s" % network) ++ return False ++ ++ elif network.version == 6: ++ if network.prefixlen < 10: ++ logging.warning("Skipping too big IP chunk: %s" % network) ++ return False ++ ++ if network.prefixlen > 48: ++ logging.info("Skipping network too small to be publicly announced: %s" % network) ++ return False ++ ++ if str(network.network_address) == "::": ++ logging.warning("Skipping network based on '::': %s" % network) ++ return False ++ ++ else: ++ # This should not happen... ++ logging.warning("Skipping network of unknown family, this should not happen: %s" % network) ++ return False ++ ++ # In case we have made it here, the network is considered to ++ # be suitable for libloc consumption... ++ return True ++ + def _parse_block(self, block): + # Get first line to find out what type of block this is + line = block[0] +@@ -559,22 +622,7 @@ class CLI(object): + + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) + +- # Bail out in case we have processed a network covering the entire IP range, which +- # is necessary to work around faulty (?) IPv6 network processing +- if network.prefixlen == 0: +- logging.warning("Skipping network covering the entire IP adress range: %s" % network) +- return +- +- # Bail out in case we have processed a network whose prefix length indicates it is +- # not globally routable (we have decided not to process them at the moment, as they +- # significantly enlarge our database without providing very helpful additional information) +- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): +- logging.info("Skipping network too small to be publicly announced: %s" % network) +- return +- +- # Bail out in case we have processed a non-public IP network +- if network.is_private: +- logging.warning("Skipping non-globally routable network: %s" % network) ++ if not self._check_parsed_network(network): + return + + self.db.execute("INSERT INTO _rirdata(network, country) \ +@@ -658,6 +706,9 @@ class CLI(object): + log.warning("Invalid IP address: %s" % address) + return + ++ if not self._check_parsed_network(network): ++ return ++ + self.db.execute("INSERT INTO networks(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO \ + UPDATE SET country = excluded.country", +-- +2.20.1 + +From eee65490a10e0fe89b3834b8be176fc900084fa0 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Wed, 21 Oct 2020 14:47:40 +0000 +Subject: [PATCH 24/70] importer.py: fetch LACNIC data via HTTPS +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Peter Müller +Signed-off-by: Michael Tremer +--- + src/python/importer.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/python/importer.py b/src/python/importer.py +index f19db4b..5f46bc3 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -64,7 +64,7 @@ EXTENDED_SOURCES = ( + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", + + # Latin America and Caribbean Network Information Centre +- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", ++ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", + + # Réseaux IP Européens + #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", +-- +2.20.1 + +From 84187ab5436eb158529d6f5e2a38890b4af3ddb4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Wed, 21 Oct 2020 14:47:41 +0000 +Subject: [PATCH 25/70] location-importer.in: omit historic/orphaned RIR data +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Some RIRs include detailled information regarding networks not managed +by or allocated to themselves, particually APNIC. We need to filter +those networks (they usually have a characteristic network name) in +order to prevent operational quirks or returning wrong country codes. + +Fixes: #12501 +Partially fixes: #12499 + +Cc: Michael Tremer +Signed-off-by: Peter Müller +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 38 +++++++++++++++++++++------------ + 1 file changed, 24 insertions(+), 14 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index f24d357..a869256 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -494,38 +494,38 @@ class CLI(object): + return False + + if not network.is_global: +- logging.warning("Skipping non-globally routable network: %s" % network) ++ log.warning("Skipping non-globally routable network: %s" % network) + return False + + if network.version == 4: + if network.prefixlen < 7: +- logging.warning("Skipping too big IP chunk: %s" % network) ++ log.warning("Skipping too big IP chunk: %s" % network) + return False + + if network.prefixlen > 24: +- logging.info("Skipping network too small to be publicly announced: %s" % network) ++ log.info("Skipping network too small to be publicly announced: %s" % network) + return False + + if str(network.network_address) == "0.0.0.0": +- logging.warning("Skipping network based on 0.0.0.0: %s" % network) ++ log.warning("Skipping network based on 0.0.0.0: %s" % network) + return False + + elif network.version == 6: + if network.prefixlen < 10: +- logging.warning("Skipping too big IP chunk: %s" % network) ++ log.warning("Skipping too big IP chunk: %s" % network) + return False + + if network.prefixlen > 48: +- logging.info("Skipping network too small to be publicly announced: %s" % network) ++ log.info("Skipping network too small to be publicly announced: %s" % network) + return False + + if str(network.network_address) == "::": +- logging.warning("Skipping network based on '::': %s" % network) ++ log.warning("Skipping network based on '::': %s" % network) + return False + + else: + # This should not happen... +- logging.warning("Skipping network of unknown family, this should not happen: %s" % network) ++ log.warning("Skipping network of unknown family, this should not happen: %s" % network) + return False + + # In case we have made it here, the network is considered to +@@ -574,15 +574,22 @@ class CLI(object): + ) + + def _parse_inetnum_block(self, block): +- logging.debug("Parsing inetnum block:") ++ log.debug("Parsing inetnum block:") + + inetnum = {} + for line in block: +- logging.debug(line) ++ log.debug(line) + + # Split line + key, val = split_line(line) + ++ # Filter any inetnum records which are only referring to IP space ++ # not managed by that specific RIR... ++ if key == "netname": ++ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()): ++ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip()) ++ return ++ + if key == "inetnum": + start_address, delim, end_address = val.partition("-") + +@@ -594,7 +601,7 @@ class CLI(object): + start_address = ipaddress.ip_address(start_address) + end_address = ipaddress.ip_address(end_address) + except ValueError: +- logging.warning("Could not parse line: %s" % line) ++ log.warning("Could not parse line: %s" % line) + return + + # Set prefix to default +@@ -611,15 +618,18 @@ class CLI(object): + inetnum[key] = val + + elif key == "country": +- if val == "UNITED STATES": +- val = "US" +- + inetnum[key] = val.upper() + + # Skip empty objects + if not inetnum or not "country" in inetnum: + return + ++ # Skip objects with bogus country code 'ZZ' ++ if inetnum.get("country") == "ZZ": ++ log.warning("Skipping network with bogus country 'ZZ': %s" % \ ++ (inetnum.get("inet6num") or inetnum.get("inetnum"))) ++ return ++ + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) + + if not self._check_parsed_network(network): +-- +2.20.1 + +From ebb087cfa30ec5ca0c96dcce66a91245c1ffc271 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Wed, 21 Oct 2020 14:47:43 +0000 +Subject: [PATCH 26/70] location-importer.in: avoid log spam for too small + networks +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Peter Müller +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index a869256..864eab1 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -503,7 +503,7 @@ class CLI(object): + return False + + if network.prefixlen > 24: +- log.info("Skipping network too small to be publicly announced: %s" % network) ++ log.debug("Skipping network too small to be publicly announced: %s" % network) + return False + + if str(network.network_address) == "0.0.0.0": +@@ -516,7 +516,7 @@ class CLI(object): + return False + + if network.prefixlen > 48: +- log.info("Skipping network too small to be publicly announced: %s" % network) ++ log.debug("Skipping network too small to be publicly announced: %s" % network) + return False + + if str(network.network_address) == "::": +-- +2.20.1 + +From bbed1fd2330e8efa6b413dc152a1a6ef2d771aac Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 27 Oct 2020 17:14:30 +0000 +Subject: [PATCH 27/70] export: Flatten the tree before exporting it + +This patch removes the possibility that any IP address ranges +might show up in multiple exported files. + +If this was content from the database: + + * 10.0.0.0/16 - DE + * 10.0.1.0/24 - FR + +Then the IP address 10.0.1.1 would match for both countries. + +The algorithm will now break the larger /16 subnet apart into +smaller subnets so that 10.0.1.0/24 is no longer overlapped. + +There was some time spent on this to make this as efficient +as possible. + +Signed-off-by: Michael Tremer +--- + src/python/export.py | 154 ++++++++++++++++++++++++++++++------------- + 1 file changed, 110 insertions(+), 44 deletions(-) + +diff --git a/src/python/export.py b/src/python/export.py +index 5eaf43f..dd44332 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -89,28 +89,55 @@ class OutputWriter(object): + + def write(self, network, subnets): + if self.flatten and self._flatten(network): +- log.debug("Skipping writing network %s" % network) ++ log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network)) + return + ++ # Convert network into a Python object ++ _network = ipaddress.ip_network("%s" % network) ++ + # Write the network when it has no subnets + if not subnets: +- network = ipaddress.ip_network("%s" % network) +- return self._write_network(network) ++ log.debug("Writing %s to %s" % (_network, self.f)) ++ return self._write_network(_network) ++ ++ # Convert subnets into Python objects ++ _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets] ++ ++ # Split the network into smaller bits so that ++ # we can accomodate for any gaps in it later ++ to_check = set() ++ for _subnet in _subnets: ++ to_check.update( ++ _network.address_exclude(_subnet) ++ ) ++ ++ # Clear the list of all subnets ++ subnets = [] ++ ++ # Check if all subnets to not overlap with anything else ++ while to_check: ++ subnet_to_check = to_check.pop() + +- # Collect all matching subnets +- matching_subnets = [] ++ for _subnet in _subnets: ++ # Drop this subnet if it equals one of the subnets ++ # or if it is subnet of one of them ++ if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet): ++ break + +- for subnet in sorted(subnets): +- # Try to find the subnet in the database +- n = self.db.lookup("%s" % subnet.network_address) ++ # Break it down if it overlaps ++ if subnet_to_check.overlaps(_subnet): ++ to_check.update( ++ subnet_to_check.address_exclude(_subnet) ++ ) ++ break + +- # No entry or matching country means those IP addresses belong here +- if not n or n.country_code == network.country_code: +- matching_subnets.append(subnet) ++ # Add the subnet again as it passed the check ++ else: ++ subnets.append(subnet_to_check) + + # Write all networks as compact as possible +- for network in ipaddress.collapse_addresses(matching_subnets): +- log.debug("Writing %s to database" % network) ++ for network in ipaddress.collapse_addresses(subnets): ++ log.debug("Writing %s to %s" % (network, self.f)) + self._write_network(network) + + def finish(self): +@@ -206,40 +233,40 @@ class Exporter(object): + # Get all networks that match the family + networks = self.db.search_networks(family=family) + +- # Materialise the generator into a list (uses quite some memory) +- networks = list(networks) ++ # Create a stack with all networks in order where we can put items back ++ # again and retrieve them in the next iteration. ++ networks = BufferedStack(networks) + + # Walk through all networks +- for i, network in enumerate(networks): +- _network = ipaddress.ip_network("%s" % network) +- +- # Search for all subnets +- subnets = set() +- +- while i < len(networks): +- subnet = networks[i+1] +- +- if subnet.is_subnet_of(network): +- _subnet = ipaddress.ip_network("%s" % subnet) +- +- subnets.add(_subnet) +- subnets.update(_network.address_exclude(_subnet)) +- +- i += 1 +- else: ++ for network in networks: ++ # Collect all networks which are a subnet of network ++ subnets = [] ++ for subnet in networks: ++ # If the next subnet was not a subnet, we have to push ++ # it back on the stack and break this loop ++ if not subnet.is_subnet_of(network): ++ networks.push(subnet) + break + ++ subnets.append(subnet) ++ + # Write matching countries +- try: +- writers[network.country_code].write(network, subnets) +- except KeyError: +- pass ++ if network.country_code and network.country_code in writers: ++ # Mismatching subnets ++ gaps = [ ++ subnet for subnet in subnets if not network.country_code == subnet.country_code ++ ] ++ ++ writers[network.country_code].write(network, gaps) + + # Write matching ASNs +- try: +- writers[network.asn].write(network, subnets) +- except KeyError: +- pass ++ if network.asn and network.asn in writers: ++ # Mismatching subnets ++ gaps = [ ++ subnet for subnet in subnets if not network.asn == subnet.asn ++ ] ++ ++ writers[network.asn].write(network, gaps) + + # Handle flags + for flag in flags: +@@ -247,10 +274,19 @@ class Exporter(object): + # Fetch the "fake" country code + country = flags[flag] + +- try: +- writers[country].write(network, subnets) +- except KeyError: +- pass ++ if not country in writers: ++ continue ++ ++ gaps = [ ++ subnet for subnet in subnets ++ if not subnet.has_flag(flag) ++ ] ++ ++ writers[country].write(network, gaps) ++ ++ # Push all subnets back onto the stack ++ for subnet in reversed(subnets): ++ networks.push(subnet) + + # Write everything to the filesystem + for writer in writers.values(): +@@ -262,3 +298,33 @@ class Exporter(object): + ) + + return os.path.join(directory, filename) ++ ++ ++class BufferedStack(object): ++ """ ++ This class takes an iterator and when being iterated ++ over it returns objects from that iterator for as long ++ as there are any. ++ ++ It additionally has a function to put an item back on ++ the back so that it will be returned again at the next ++ iteration. ++ """ ++ def __init__(self, iterator): ++ self.iterator = iterator ++ self.stack = [] ++ ++ def __iter__(self): ++ return self ++ ++ def __next__(self): ++ if self.stack: ++ return self.stack.pop(0) ++ ++ return next(self.iterator) ++ ++ def push(self, elem): ++ """ ++ Takes an element and puts it on the stack ++ """ ++ self.stack.insert(0, elem) +-- +2.20.1 + +From e99a72265c1ba2194b61663eda7e9f14e0083016 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 28 Oct 2020 09:52:36 +0000 +Subject: [PATCH 28/70] location: Fix Python syntax error in verify() + +The database is now being opened before the requested +method is called and handle_verify() wasn't updated. + +Signed-off-by: Michael Tremer +--- + src/python/location.in | 8 +------- + 1 file changed, 1 insertion(+), 7 deletions(-) + +diff --git a/src/python/location.in b/src/python/location.in +index 44ad726..b5e5758 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -453,13 +453,7 @@ class CLI(object): + + return 0 + +- def handle_verify(self, ns): +- try: +- db = location.Database(ns.database) +- except FileNotFoundError as e: +- log.error("%s: %s" % (ns.database, e)) +- return 127 +- ++ def handle_verify(self, db, ns): + # Verify the database + with open(ns.public_key, "r") as f: + if not db.verify(f): +-- +2.20.1 + +From 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Thu, 29 Oct 2020 07:25:53 -0700 +Subject: [PATCH 29/70] location update: Remove double conversion of timestamps +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This caused that the timestamp in DNS was misinterpreted +as local time and often, databases could not be downloaded. + +Signed-off-by: Peter Müller +--- + src/python/location.in | 5 +---- + 1 file changed, 1 insertion(+), 4 deletions(-) + +diff --git a/src/python/location.in b/src/python/location.in +index b5e5758..070640c 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -421,11 +421,8 @@ class CLI(object): + # Fetch the timestamp we need from DNS + t = location.discover_latest_version() + +- # Parse timestamp into datetime format +- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None +- + # Check the version of the local database +- if db and timestamp and db.created_at >= timestamp.timestamp(): ++ if db and t and db.created_at >= t: + log.info("Already on the latest version") + return + +-- +2.20.1 + +From 60c1ac0307312614bd6980d30b44bb59b5a6ab6e Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Thu, 29 Oct 2020 07:36:46 -0700 +Subject: [PATCH 30/70] location.in: do not confuse UTC with local time zones +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Peter Müller +--- + src/python/location.in | 12 +++++------- + 1 file changed, 5 insertions(+), 7 deletions(-) + +diff --git a/src/python/location.in b/src/python/location.in +index 070640c..0d09210 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -398,10 +398,7 @@ class CLI(object): + + def handle_update(self, db, ns): + if ns.cron and db: +- now = datetime.datetime.utcnow() +- +- # Parse the database timestamp +- t = datetime.datetime.utcfromtimestamp(db.created_at) ++ now = time.time() + + if ns.cron == "daily": + delta = datetime.timedelta(days=1) +@@ -410,11 +407,12 @@ class CLI(object): + elif ns.cron == "monthly": + delta = datetime.timedelta(days=30) + ++ delta = delta.total_seconds() ++ + # Check if the database has recently been updated +- if t >= (now - delta): ++ if db.created_at >= (now - delta): + log.info( +- _("The database has been updated recently (%s)") % \ +- format_timedelta(now - t), ++ _("The database has been updated recently"), + ) + return 3 + +-- +2.20.1 + +From e7d612e5219ef9ba612ed404e4e2c174110d3dd7 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= +Date: Tue, 3 Nov 2020 15:31:08 +0000 +Subject: [PATCH 31/70] location-importer.in: always convert organisation + handles into upper cases +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fixes: #12523 + +Signed-off-by: Peter Müller +Signed-off-by: Michael Tremer +--- + src/python/location-importer.in | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 864eab1..2dec89e 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -560,7 +560,7 @@ class CLI(object): + autnum["asn"] = m.group(2) + + elif key == "org": +- autnum[key] = val ++ autnum[key] = val.upper() + + # Skip empty objects + if not autnum: +@@ -646,7 +646,9 @@ class CLI(object): + # Split line + key, val = split_line(line) + +- if key in ("organisation", "org-name"): ++ if key == "organisation": ++ org[key] = val.upper() ++ elif key == "org-name": + org[key] = val + + # Skip empty objects +-- +2.20.1 + +From e96704f43acca1a8f56d9a680cce281f5e587ec5 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 11 Nov 2020 21:16:45 +0000 +Subject: [PATCH 32/70] test: Add tests for database enumerator + +Signed-off-by: Michael Tremer +--- + src/test-database.c | 53 +++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 53 insertions(+) + +diff --git a/src/test-database.c b/src/test-database.c +index b4a75c4..4aef94e 100644 +--- a/src/test-database.c ++++ b/src/test-database.c +@@ -38,6 +38,14 @@ const char* DESCRIPTION = + "Maecenas ut venenatis nunc."; + const char* LICENSE = "CC"; + ++const char* networks[] = { ++ "2001:db8::/32", ++ "2001:db8:1000::/48", ++ "2001:db8:2000::/48", ++ "2001:db8:2020::/48", ++ NULL, ++}; ++ + static int attempt_to_open(struct loc_ctx* ctx, char* path) { + FILE* f = fopen(path, "r"); + if (!f) +@@ -139,6 +147,24 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++ struct loc_network* network = NULL; ++ ++ // Add some networks ++ const char** n = networks; ++ while (*n) { ++ err = loc_writer_add_network(writer, &network, *n); ++ if (err) { ++ fprintf(stderr, "Could not add network %s\n", *n); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Set a country ++ loc_network_set_country_code(network, "XX"); ++ ++ // Next one ++ n++; ++ } ++ + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +@@ -170,6 +196,33 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++ // Enumerator ++ struct loc_database_enumerator* enumerator; ++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS); ++ if (err) { ++ fprintf(stderr, "Could not initialise the enumerator: %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Walk through all networks ++ while (1) { ++ err = loc_database_enumerator_next_network(enumerator, &network); ++ if (err) { ++ fprintf(stderr, "Error fetching the next network: %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (!network) ++ break; ++ ++ char* s = loc_network_str(network); ++ printf("Got network: %s\n", s); ++ free(s); ++ } ++ ++ // Free the enumerator ++ loc_database_enumerator_unref(enumerator); ++ + // Close the database + loc_database_unref(db); + loc_unref(ctx); +-- +2.20.1 + +From ecce288da39a2c0eb60076050ca21e9619f61844 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 11 Nov 2020 23:01:19 +0000 +Subject: [PATCH 33/70] networks: Add list to manage groups of networks + +Signed-off-by: Michael Tremer +--- + src/libloc.sym | 11 ++++++ + src/loc/network.h | 12 ++++++ + src/network.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 119 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index b8296eb..6ef2d27 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -98,6 +98,17 @@ global: + loc_network_str; + loc_network_unref; + ++ # Network List ++ loc_network_list_clear; ++ loc_network_list_empty; ++ loc_network_list_get; ++ loc_network_list_new; ++ loc_network_list_pop; ++ loc_network_list_push; ++ loc_network_list_ref; ++ loc_network_list_size; ++ loc_network_list_unref; ++ + # Writer + loc_writer_add_as; + loc_writer_add_country; +diff --git a/src/loc/network.h b/src/loc/network.h +index 70c3803..fd20812 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -56,6 +56,18 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag); + + int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); + ++// List ++struct loc_network_list; ++int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); ++struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); ++struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); ++size_t loc_network_list_size(struct loc_network_list* list); ++int loc_network_list_empty(struct loc_network_list* list); ++void loc_network_list_clear(struct loc_network_list* list); ++struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); ++int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); ++struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++ + #ifdef LIBLOC_PRIVATE + + int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj); +diff --git a/src/network.c b/src/network.c +index d7b1645..c9e7979 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -746,3 +746,99 @@ LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) + LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { + return loc_network_ref(node->network); + } ++ ++// List ++ ++struct loc_network_list { ++ struct loc_ctx* ctx; ++ int refcount; ++ ++ struct loc_network* list[1024]; ++ size_t size; ++ size_t max_size; ++}; ++ ++LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, ++ struct loc_network_list** list) { ++ struct loc_network_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; ++ ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; ++ ++ // Do not allow this list to grow larger than this ++ l->max_size = 1024; ++ ++ DEBUG(l->ctx, "Network list allocated at %p\n", l); ++ *list = l; ++ return 0; ++} ++ ++LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { ++ list->refcount++; ++ ++ return list; ++} ++ ++static void loc_network_list_free(struct loc_network_list* list) { ++ DEBUG(list->ctx, "Releasing network list at %p\n", list); ++ ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_network_unref(list->list[i]); ++ ++ loc_unref(list->ctx); ++ free(list); ++} ++ ++LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { ++ if (!list) ++ return NULL; ++ ++ if (--list->refcount > 0) ++ return list; ++ ++ loc_network_list_free(list); ++ return NULL; ++} ++ ++LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { ++ return list->size; ++} ++ ++LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { ++ return list->size == 0; ++} ++ ++LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_network_unref(list->list[i]); ++ ++ list->size = 0; ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; ++ ++ return loc_network_ref(list->list[index]); ++} ++ ++LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { ++ // Check if we have space left ++ if (list->size == list->max_size) ++ return -ENOMEM; ++ ++ list->list[list->size++] = loc_network_ref(network); ++ ++ return 0; ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) ++ return NULL; ++ ++ return list->list[list->size--]; ++} +-- +2.20.1 + +From 8b2205272b7872a1458ad87811abf58609f38ad4 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 13:57:35 +0000 +Subject: [PATCH 34/70] networks: Add function to dump lists + +Signed-off-by: Michael Tremer +--- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 14 ++++++++++++++ + 3 files changed, 16 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index 6ef2d27..a5641c6 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -100,6 +100,7 @@ global: + + # Network List + loc_network_list_clear; ++ loc_network_list_dump; + loc_network_list_empty; + loc_network_list_get; + loc_network_list_new; +diff --git a/src/loc/network.h b/src/loc/network.h +index fd20812..44c50a4 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -64,6 +64,7 @@ struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); + size_t loc_network_list_size(struct loc_network_list* list); + int loc_network_list_empty(struct loc_network_list* list); + void loc_network_list_clear(struct loc_network_list* list); ++void loc_network_list_dump(struct loc_network_list* list); + struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); + int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); + struct loc_network* loc_network_list_pop(struct loc_network_list* list); +diff --git a/src/network.c b/src/network.c +index c9e7979..0977406 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -817,6 +817,20 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { + list->size = 0; + } + ++LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { ++ struct loc_network* network; ++ char* s; ++ ++ for (unsigned int i = 0; i < list->size; i++) { ++ network = list->list[i]; ++ ++ s = loc_network_str(network); ++ ++ INFO(list->ctx, "%s\n", s); ++ free(s); ++ } ++} ++ + LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { + // Check index + if (index >= list->size) +-- +2.20.1 + +From 850e75167e8e03fe8b951992c9f7bc2ccb9fb711 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 14:18:40 +0000 +Subject: [PATCH 35/70] network: Add functions to break network into subnets + +Signed-off-by: Michael Tremer +--- + src/libloc.sym | 5 + + src/loc/network.h | 6 + + src/network.c | 290 ++++++++++++++++++++++++++++++++++++++++++++- + src/test-network.c | 61 ++++++++++ + 4 files changed, 361 insertions(+), 1 deletion(-) + +diff --git a/src/libloc.sym b/src/libloc.sym +index a5641c6..fcb9ea5 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -80,6 +80,8 @@ global: + + # Network + loc_network_address_family; ++ loc_network_eq; ++ loc_network_exclude; + loc_network_format_first_address; + loc_network_format_last_address; + loc_network_get_asn; +@@ -96,6 +98,7 @@ global: + loc_network_set_country_code; + loc_network_set_flag; + loc_network_str; ++ loc_network_subnets; + loc_network_unref; + + # Network List +@@ -107,7 +110,9 @@ global: + loc_network_list_pop; + loc_network_list_push; + loc_network_list_ref; ++ loc_network_list_reverse; + loc_network_list_size; ++ loc_network_list_sort; + loc_network_list_unref; + + # Writer +diff --git a/src/loc/network.h b/src/loc/network.h +index 44c50a4..4e51a62 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -54,7 +54,11 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag); + int loc_network_set_flag(struct loc_network* network, uint32_t flag); + int loc_network_match_flag(struct loc_network* network, uint32_t flag); + ++int loc_network_eq(struct loc_network* self, struct loc_network* other); + int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); ++struct loc_network_list* loc_network_subnets(struct loc_network* network); ++struct loc_network_list* loc_network_exclude( ++ struct loc_network* self, struct loc_network* other); + + // List + struct loc_network_list; +@@ -68,6 +72,8 @@ void loc_network_list_dump(struct loc_network_list* list); + struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); + int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); + struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++void loc_network_list_sort(struct loc_network_list* list); ++void loc_network_list_reverse(struct loc_network_list* list); + + #ifdef LIBLOC_PRIVATE + +diff --git a/src/network.c b/src/network.c +index 0977406..6c08070 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -97,6 +97,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s + return a; + } + ++static struct in6_addr address_increment(const struct in6_addr* address) { ++ struct in6_addr a = *address; ++ ++ for (int octet = 15; octet >= 0; octet--) { ++ if (a.s6_addr[octet] < 255) { ++ a.s6_addr[octet]++; ++ break; ++ } else { ++ a.s6_addr[octet] = 0; ++ } ++ } ++ ++ return a; ++} ++ + LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network, + struct in6_addr* address, unsigned int prefix) { + // Address cannot be unspecified +@@ -405,6 +420,69 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag + return loc_network_has_flag(network, flag); + } + ++LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { ++#ifdef ENABLE_DEBUG ++ char* n1 = loc_network_str(self); ++ char* n2 = loc_network_str(other); ++ ++ DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2); ++ ++ free(n1); ++ free(n2); ++#endif ++ ++ // Family must be the same ++ if (self->family != other->family) { ++ DEBUG(self->ctx, " Family mismatch\n"); ++ ++ return 0; ++ } ++ ++ // The start address must be the same ++ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) { ++ DEBUG(self->ctx, " Address mismatch\n"); ++ ++ return 0; ++ } ++ ++ // The prefix length must be the same ++ if (self->prefix != other->prefix) { ++ DEBUG(self->ctx, " Prefix mismatch\n"); ++ return 0; ++ } ++ ++ DEBUG(self->ctx, " Yes!\n"); ++ ++ return 1; ++} ++ ++static int loc_network_gt(struct loc_network* self, struct loc_network* other) { ++ // Families must match ++ if (self->family != other->family) ++ return -1; ++ ++ int r = in6_addr_cmp(&self->first_address, &other->first_address); ++ ++ switch (r) { ++ // Smaller ++ case -1: ++ return 0; ++ ++ // Larger ++ case 1: ++ return 1; ++ ++ default: ++ break; ++ } ++ ++ if (self->prefix > other->prefix) ++ return 1; ++ ++ // Dunno ++ return 0; ++} ++ + LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { + // If the start address of the other network is smaller than this network, + // it cannot be a subnet. +@@ -419,6 +497,175 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net + return 1; + } + ++LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) { ++ struct loc_network_list* list; ++ ++ // New prefix length ++ unsigned int prefix = network->prefix + 1; ++ ++ // Check if the new prefix is valid ++ if (valid_prefix(&network->first_address, prefix)) ++ return NULL; ++ ++ // Create a new list with the result ++ int r = loc_network_list_new(network->ctx, &list); ++ if (r) { ++ ERROR(network->ctx, "Could not create network list: %d\n", r); ++ return NULL; ++ } ++ ++ struct loc_network* subnet1 = NULL; ++ struct loc_network* subnet2 = NULL; ++ ++ // Create the first half of the network ++ r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix); ++ if (r) ++ goto ERROR; ++ ++ // The next subnet starts after the first one ++ struct in6_addr first_address = address_increment(&subnet1->last_address); ++ ++ // Create the second half of the network ++ r = loc_network_new(network->ctx, &subnet2, &first_address, prefix); ++ if (r) ++ goto ERROR; ++ ++ // Push the both onto the stack (in reverse order) ++ r = loc_network_list_push(list, subnet2); ++ if (r) ++ goto ERROR; ++ ++ r = loc_network_list_push(list, subnet1); ++ if (r) ++ goto ERROR; ++ ++ loc_network_unref(subnet1); ++ loc_network_unref(subnet2); ++ ++ return list; ++ ++ERROR: ++ if (subnet1) ++ loc_network_unref(subnet1); ++ ++ if (subnet2) ++ loc_network_unref(subnet2); ++ ++ if (list) ++ loc_network_list_unref(list); ++ ++ return NULL; ++} ++ ++LOC_EXPORT struct loc_network_list* loc_network_exclude( ++ struct loc_network* self, struct loc_network* other) { ++ struct loc_network_list* list; ++ ++#ifdef ENABLE_DEBUG ++ char* n1 = loc_network_str(self); ++ char* n2 = loc_network_str(other); ++ ++ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); ++ ++ free(n1); ++ free(n2); ++#endif ++ ++ // Family must match ++ if (self->family != other->family) { ++ DEBUG(self->ctx, "Family mismatch\n"); ++ ++ return NULL; ++ } ++ ++ // Other must be a subnet of self ++ if (!loc_network_is_subnet_of(other, self)) { ++ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); ++ ++ return NULL; ++ } ++ ++ // We cannot perform this operation if both networks equal ++ if (loc_network_eq(self, other)) { ++ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); ++ ++ return NULL; ++ } ++ ++ // Create a new list with the result ++ int r = loc_network_list_new(self->ctx, &list); ++ if (r) { ++ ERROR(self->ctx, "Could not create network list: %d\n", r); ++ return NULL; ++ } ++ ++ struct loc_network_list* subnets = loc_network_subnets(self); ++ ++ struct loc_network* subnet1 = NULL; ++ struct loc_network* subnet2 = NULL; ++ ++ while (subnets) { ++ // Fetch both subnets ++ subnet1 = loc_network_list_get(subnets, 0); ++ subnet2 = loc_network_list_get(subnets, 1); ++ ++ // Free list ++ loc_network_list_unref(subnets); ++ subnets = NULL; ++ ++ if (loc_network_eq(other, subnet1)) { ++ r = loc_network_list_push(list, subnet2); ++ if (r) ++ goto ERROR; ++ ++ } else if (loc_network_eq(other, subnet2)) { ++ r = loc_network_list_push(list, subnet1); ++ if (r) ++ goto ERROR; ++ ++ } else if (loc_network_is_subnet_of(other, subnet1)) { ++ r = loc_network_list_push(list, subnet2); ++ if (r) ++ goto ERROR; ++ ++ subnets = loc_network_subnets(subnet1); ++ ++ } else if (loc_network_is_subnet_of(other, subnet2)) { ++ r = loc_network_list_push(list, subnet1); ++ if (r) ++ goto ERROR; ++ ++ subnets = loc_network_subnets(subnet2); ++ ++ } else { ++ ERROR(self->ctx, "We should never get here\n"); ++ goto ERROR; ++ } ++ ++ loc_network_unref(subnet1); ++ loc_network_unref(subnet2); ++ } ++ ++#ifdef ENABLE_DEBUG ++ loc_network_list_dump(list); ++#endif ++ ++ // Return the result ++ return list; ++ ++ERROR: ++ if (subnet1) ++ loc_network_unref(subnet1); ++ ++ if (subnet2) ++ loc_network_unref(subnet2); ++ ++ if (list) ++ loc_network_list_unref(list); ++ ++ return NULL; ++} ++ + LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { + // Add country code + loc_country_code_copy(dbobj->country_code, network->country_code); +@@ -854,5 +1101,46 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis + if (loc_network_list_empty(list)) + return NULL; + +- return list->list[list->size--]; ++ return list->list[--list->size]; ++} ++ ++static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { ++ // Do nothing for invalid indices ++ if (i1 >= list->size || i2 >= list->size) ++ return; ++ ++ DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2); ++ ++ struct loc_network* network1 = list->list[i1]; ++ struct loc_network* network2 = list->list[i2]; ++ ++ list->list[i1] = network2; ++ list->list[i2] = network1; ++} ++ ++LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { ++ unsigned int i = 0; ++ unsigned int j = list->size - 1; ++ ++ while (i < j) { ++ loc_network_list_swap(list, i++, j--); ++ } ++} ++ ++LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { ++ unsigned int n = list->size; ++ int swapped; ++ ++ do { ++ swapped = 0; ++ ++ for (unsigned int i = 1; i < n; i++) { ++ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { ++ loc_network_list_swap(list, i-1, i); ++ swapped = 1; ++ } ++ } ++ ++ n--; ++ } while (swapped); + } +diff --git a/src/test-network.c b/src/test-network.c +index b6776b4..af1b2e6 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -118,6 +118,19 @@ int main(int argc, char** argv) { + size_t nodes = loc_network_tree_count_nodes(tree); + printf("The tree has %zu nodes\n", nodes); + ++ // Check equals function ++ err = loc_network_eq(network1, network1); ++ if (!err) { ++ fprintf(stderr, "Network is not equal with itself\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ err = loc_network_eq(network1, network2); ++ if (err) { ++ fprintf(stderr, "Networks equal unexpectedly\n"); ++ exit(EXIT_FAILURE); ++ } ++ + // Check subnet function + err = loc_network_is_subnet_of(network1, network2); + if (err != 0) { +@@ -131,6 +144,54 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++ // Make list of subnets ++ struct loc_network_list* subnets = loc_network_subnets(network1); ++ if (!subnets) { ++ fprintf(stderr, "Could not find subnets of network\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ loc_network_list_dump(subnets); ++ ++ while (!loc_network_list_empty(subnets)) { ++ struct loc_network* subnet = loc_network_list_pop(subnets); ++ if (!subnet) { ++ fprintf(stderr, "Received an empty subnet\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ char* s = loc_network_str(subnet); ++ printf("Received subnet %s\n", s); ++ free(s); ++ ++ if (!loc_network_is_subnet_of(subnet, network1)) { ++ fprintf(stderr, "Not a subnet\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ loc_network_unref(subnet); ++ } ++ ++ loc_network_list_unref(subnets); ++ ++ struct loc_network_list* excluded = loc_network_exclude(network1, network2); ++ if (!excluded) { ++ fprintf(stderr, "Could not create excluded list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ loc_network_list_dump(excluded); ++ ++ // Reverse it ++ loc_network_list_reverse(excluded); ++ loc_network_list_dump(excluded); ++ ++ // Sort them and dump them again ++ loc_network_list_sort(excluded); ++ loc_network_list_dump(excluded); ++ ++ loc_network_list_unref(excluded); ++ + // Create a database + struct loc_writer* writer; + err = loc_writer_new(ctx, &writer, NULL, NULL); +-- +2.20.1 + +From 6159d384c4a98fe45ec52522e2950719e4982d80 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 14:24:58 +0000 +Subject: [PATCH 36/70] networks: Add function to check if two networks overlap + +Signed-off-by: Michael Tremer +--- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 16 ++++++++++++++++ + 3 files changed, 18 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index fcb9ea5..0c2835b 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -93,6 +93,7 @@ global: + loc_network_match_flag; + loc_network_new; + loc_network_new_from_string; ++ loc_network_overlaps; + loc_network_ref; + loc_network_set_asn; + loc_network_set_country_code; +diff --git a/src/loc/network.h b/src/loc/network.h +index 4e51a62..ef13756 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -55,6 +55,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); + int loc_network_match_flag(struct loc_network* network, uint32_t flag); + + int loc_network_eq(struct loc_network* self, struct loc_network* other); ++int loc_network_overlaps(struct loc_network* self, struct loc_network* other); + int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); + struct loc_network_list* loc_network_subnets(struct loc_network* network); + struct loc_network_list* loc_network_exclude( +diff --git a/src/network.c b/src/network.c +index 6c08070..d826511 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -483,6 +483,22 @@ static int loc_network_gt(struct loc_network* self, struct loc_network* other) { + return 0; + } + ++LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { ++ if (loc_network_match_address(self, &other->first_address) == 0) ++ return 1; ++ ++ if (loc_network_match_address(self, &other->last_address) == 0) ++ return 1; ++ ++ if (loc_network_match_address(other, &self->first_address) == 0) ++ return 1; ++ ++ if (loc_network_match_address(other, &self->last_address) == 0) ++ return 1; ++ ++ return 0; ++} ++ + LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { + // If the start address of the other network is smaller than this network, + // it cannot be a subnet. +-- +2.20.1 + +From e52ba21761f27e592040a2793b2a26bbeeeecc05 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 14:28:15 +0000 +Subject: [PATCH 37/70] networks: Add function to check if network is part of a + list + +Signed-off-by: Michael Tremer +--- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 9 +++++++++ + 3 files changed, 11 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index 0c2835b..f1b63a2 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -104,6 +104,7 @@ global: + + # Network List + loc_network_list_clear; ++ loc_network_list_contains; + loc_network_list_dump; + loc_network_list_empty; + loc_network_list_get; +diff --git a/src/loc/network.h b/src/loc/network.h +index ef13756..7804512 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -73,6 +73,7 @@ void loc_network_list_dump(struct loc_network_list* list); + struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); + int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); + struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); + void loc_network_list_sort(struct loc_network_list* list); + void loc_network_list_reverse(struct loc_network_list* list); + +diff --git a/src/network.c b/src/network.c +index d826511..fcbdc59 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -1120,6 +1120,15 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis + return list->list[--list->size]; + } + ++LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_network_eq(list->list[i], network)) ++ return 1; ++ } ++ ++ return 0; ++} ++ + static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { + // Do nothing for invalid indices + if (i1 >= list->size || i2 >= list->size) +-- +2.20.1 + +From f802f3a4decf4827ecc8bcabe269ae9f94f7f32d Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 14:33:22 +0000 +Subject: [PATCH 38/70] networks: Add function to merge two lists + +Signed-off-by: Michael Tremer +--- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 13 +++++++++++++ + 3 files changed, 15 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index f1b63a2..c0b6b1f 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -108,6 +108,7 @@ global: + loc_network_list_dump; + loc_network_list_empty; + loc_network_list_get; ++ loc_network_list_merge; + loc_network_list_new; + loc_network_list_pop; + loc_network_list_push; +diff --git a/src/loc/network.h b/src/loc/network.h +index 7804512..e30d91c 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -76,6 +76,7 @@ struct loc_network* loc_network_list_pop(struct loc_network_list* list); + int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); + void loc_network_list_sort(struct loc_network_list* list); + void loc_network_list_reverse(struct loc_network_list* list); ++int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); + + #ifdef LIBLOC_PRIVATE + +diff --git a/src/network.c b/src/network.c +index fcbdc59..541286d 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -1169,3 +1169,16 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { + n--; + } while (swapped); + } ++ ++LOC_EXPORT int loc_network_list_merge( ++ struct loc_network_list* self, struct loc_network_list* other) { ++ int r; ++ ++ for (unsigned int i = 0; i < other->size; i++) { ++ r = loc_network_list_push(self, other->list[i]); ++ if (r) ++ return r; ++ } ++ ++ return 0; ++} +-- +2.20.1 + +From 6d22a179dffd08fcf2a44aafb361725ab22486d4 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 14:35:43 +0000 +Subject: [PATCH 39/70] network: Make lists unique + +Networks that are in the list won't be added again + +Signed-off-by: Michael Tremer +--- + src/network.c | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/src/network.c b/src/network.c +index 541286d..44571b3 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -1103,6 +1103,10 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis + } + + LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { ++ // Do not add networks that are already on the list ++ if (loc_network_list_contains(list, network)) ++ return 0; ++ + // Check if we have space left + if (list->size == list->max_size) + return -ENOMEM; +-- +2.20.1 + +From 681ff05cb7cdf230d38abf09a330a31498e265a4 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 19:21:13 +0000 +Subject: [PATCH 40/70] database: Pass flag to enumerator to flatten output + +Signed-off-by: Michael Tremer +--- + src/database.c | 8 +++++++- + src/loc/database.h | 6 +++++- + src/perl/Location.xs | 2 +- + src/python/database.c | 25 ++++++++++++++++++------- + 4 files changed, 31 insertions(+), 10 deletions(-) + +diff --git a/src/database.c b/src/database.c +index fa1dad0..9baab33 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -104,6 +104,9 @@ struct loc_database_enumerator { + enum loc_network_flags flags; + int family; + ++ // Flatten output? ++ int flatten; ++ + // Index of the AS we are looking at + unsigned int as_index; + +@@ -933,7 +936,7 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, + // Enumerator + + LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, +- struct loc_database* db, enum loc_database_enumerator_mode mode) { ++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { + struct loc_database_enumerator* e = calloc(1, sizeof(*e)); + if (!e) + return -ENOMEM; +@@ -944,6 +947,9 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum + e->mode = mode; + e->refcount = 1; + ++ // Flatten output? ++ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); ++ + // Initialise graph search + //e->network_stack[++e->network_stack_depth] = 0; + e->network_stack_depth = 1; +diff --git a/src/loc/database.h b/src/loc/database.h +index 43173dd..14eb5ea 100644 +--- a/src/loc/database.h ++++ b/src/loc/database.h +@@ -55,9 +55,13 @@ enum loc_database_enumerator_mode { + LOC_DB_ENUMERATE_COUNTRIES = 3, + }; + ++enum loc_database_enumerator_flags { ++ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0), ++}; ++ + struct loc_database_enumerator; + int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, +- struct loc_database* db, enum loc_database_enumerator_mode mode); ++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags); + struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator); + struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); + +diff --git a/src/perl/Location.xs b/src/perl/Location.xs +index dcf3f0d..b7676d2 100644 +--- a/src/perl/Location.xs ++++ b/src/perl/Location.xs +@@ -125,7 +125,7 @@ database_countries(db) + PPCODE: + // Create Database enumerator + struct loc_database_enumerator* enumerator; +- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES); ++ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0); + + if (err) { + croak("Could not create a database enumerator\n"); +diff --git a/src/python/database.c b/src/python/database.c +index 1013a58..7f8c2c2 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -207,10 +207,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database + return (PyObject*)self; + } + +-static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) { ++static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) { + struct loc_database_enumerator* enumerator; + +- int r = loc_database_enumerator_new(&enumerator, self->db, what); ++ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +@@ -223,7 +223,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en + } + + static PyObject* Database_ases(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0); + } + + static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { +@@ -234,7 +234,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { + + struct loc_database_enumerator* enumerator; + +- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES); ++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +@@ -250,7 +250,11 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { + } + + static PyObject* Database_networks(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0); ++} ++ ++static PyObject* Database_networks_flattened(DatabaseObject *self) { ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN); + } + + static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { +@@ -264,7 +268,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, + return NULL; + + struct loc_database_enumerator* enumerator; +- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS); ++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +@@ -317,7 +321,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, + } + + static PyObject* Database_countries(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0); + } + + static struct PyMethodDef Database_methods[] = { +@@ -403,6 +407,13 @@ static struct PyGetSetDef Database_getsetters[] = { + NULL, + NULL, + }, ++ { ++ "networks_flattened", ++ (getter)Database_networks_flattened, ++ NULL, ++ NULL, ++ NULL, ++ }, + { + "vendor", + (getter)Database_get_vendor, +-- +2.20.1 + +From f5e50a47e37e9b29d0d2ee9e5a41e5a5fe5aea7f Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 19:21:58 +0000 +Subject: [PATCH 41/70] network: Reduce debugging output + +Signed-off-by: Michael Tremer +--- + src/network.c | 28 +++------------------------- + 1 file changed, 3 insertions(+), 25 deletions(-) + +diff --git a/src/network.c b/src/network.c +index 44571b3..f7071a6 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -421,37 +421,17 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag + } + + LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { +-#ifdef ENABLE_DEBUG +- char* n1 = loc_network_str(self); +- char* n2 = loc_network_str(other); +- +- DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2); +- +- free(n1); +- free(n2); +-#endif +- + // Family must be the same +- if (self->family != other->family) { +- DEBUG(self->ctx, " Family mismatch\n"); +- ++ if (self->family != other->family) + return 0; +- } + + // The start address must be the same +- if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) { +- DEBUG(self->ctx, " Address mismatch\n"); +- ++ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) + return 0; +- } + + // The prefix length must be the same +- if (self->prefix != other->prefix) { +- DEBUG(self->ctx, " Prefix mismatch\n"); ++ if (self->prefix != other->prefix) + return 0; +- } +- +- DEBUG(self->ctx, " Yes!\n"); + + return 1; + } +@@ -1138,8 +1118,6 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1 + if (i1 >= list->size || i2 >= list->size) + return; + +- DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2); +- + struct loc_network* network1 = list->list[i1]; + struct loc_network* network2 = list->list[i2]; + +-- +2.20.1 + +From 037c65d3a07ec6d37ff063f0645adda6b483b407 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 19:36:38 +0000 +Subject: [PATCH 42/70] python: Export networks exclude function + +Signed-off-by: Michael Tremer +--- + src/python/network.c | 39 +++++++++++++++++++++++++++++++++++++++ + 1 file changed, 39 insertions(+) + +diff --git a/src/python/network.c b/src/python/network.c +index 5496d1e..11f672b 100644 +--- a/src/python/network.c ++++ b/src/python/network.c +@@ -24,6 +24,24 @@ + #include "locationmodule.h" + #include "network.h" + ++static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) { ++ PyObject* list = PyList_New(0); ++ if (!networks) ++ return list; ++ ++ while (!loc_network_list_empty(networks)) { ++ struct loc_network* network = loc_network_list_pop(networks); ++ ++ PyObject* n = new_network(&NetworkType, network); ++ PyList_Append(list, n); ++ ++ loc_network_unref(network); ++ Py_DECREF(n); ++ } ++ ++ return list; ++} ++ + PyObject* new_network(PyTypeObject* type, struct loc_network* network) { + NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0); + if (self) { +@@ -154,6 +172,21 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) { + Py_RETURN_NONE; + } + ++static PyObject* Network_exclude(NetworkObject* self, PyObject* args) { ++ NetworkObject* other = NULL; ++ ++ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) ++ return NULL; ++ ++ struct loc_network_list* list = loc_network_exclude(self->network, other->network); ++ ++ // Convert to Python objects ++ PyObject* obj = PyList_FromNetworkList(list); ++ loc_network_list_unref(list); ++ ++ return obj; ++} ++ + static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { + NetworkObject* other = NULL; + +@@ -191,6 +224,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) { + } + + static struct PyMethodDef Network_methods[] = { ++ { ++ "exclude", ++ (PyCFunction)Network_exclude, ++ METH_VARARGS, ++ NULL, ++ }, + { + "has_flag", + (PyCFunction)Network_has_flag, +-- +2.20.1 + +From 9a7732c8679e805d4d2d55ea4750c5d70ca4bd2c Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 19:59:22 +0000 +Subject: [PATCH 43/70] network: Add more debugging output to stacks + +Signed-off-by: Michael Tremer +--- + src/network.c | 16 +++++++++++++--- + 1 file changed, 13 insertions(+), 3 deletions(-) + +diff --git a/src/network.c b/src/network.c +index f7071a6..d41e873 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -1088,8 +1088,12 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n + return 0; + + // Check if we have space left +- if (list->size == list->max_size) ++ if (list->size == list->max_size) { ++ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); + return -ENOMEM; ++ } ++ ++ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); + + list->list[list->size++] = loc_network_ref(network); + +@@ -1098,10 +1102,16 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n + + LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { + // Return nothing when empty +- if (loc_network_list_empty(list)) ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); + return NULL; ++ } + +- return list->list[--list->size]; ++ struct loc_network* network = list->list[--list->size]; ++ ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++ ++ return network; + } + + LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { +-- +2.20.1 + +From 33a051e0435f6e78cc936f26f3b9ee16b7851025 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 20:00:09 +0000 +Subject: [PATCH 44/70] network: Add new subnet function + +The old one is too difficult to use in terms of order +of input parameters and return value. + +Signed-off-by: Michael Tremer +--- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 15 +++++++++++++++ + 3 files changed, 17 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index c0b6b1f..5392437 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -87,6 +87,7 @@ global: + loc_network_get_asn; + loc_network_get_country_code; + loc_network_has_flag; ++ loc_network_is_subnet; + loc_network_is_subnet_of; + loc_network_match_asn; + loc_network_match_country_code; +diff --git a/src/loc/network.h b/src/loc/network.h +index e30d91c..2154cdc 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -56,6 +56,7 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag); + + int loc_network_eq(struct loc_network* self, struct loc_network* other); + int loc_network_overlaps(struct loc_network* self, struct loc_network* other); ++int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); + int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); + struct loc_network_list* loc_network_subnets(struct loc_network* network); + struct loc_network_list* loc_network_exclude( +diff --git a/src/network.c b/src/network.c +index d41e873..5719111 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -479,6 +479,21 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network + return 0; + } + ++LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { ++ // If the start address of the other network is smaller than this network, ++ // it cannot be a subnet. ++ if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) ++ return 0; ++ ++ // If the end address of the other network is greater than this network, ++ // it cannot be a subnet. ++ if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) ++ return 0; ++ ++ return 1; ++} ++ ++// XXX DEPRECATED - I find this too difficult to use + LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { + // If the start address of the other network is smaller than this network, + // it cannot be a subnet. +-- +2.20.1 + +From add5bb652ba1dad1127f79cb6a0db2d363a6d5e5 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 20:01:17 +0000 +Subject: [PATCH 45/70] network: Add function to exclude multiple networks at + once + +Signed-off-by: Michael Tremer +--- + src/libloc.sym | 1 + + src/loc/network.h | 2 ++ + src/network.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 88 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index 5392437..bcd11be 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -82,6 +82,7 @@ global: + loc_network_address_family; + loc_network_eq; + loc_network_exclude; ++ loc_network_exclude_list; + loc_network_format_first_address; + loc_network_format_last_address; + loc_network_get_asn; +diff --git a/src/loc/network.h b/src/loc/network.h +index 2154cdc..40712b9 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -61,6 +61,8 @@ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other + struct loc_network_list* loc_network_subnets(struct loc_network* network); + struct loc_network_list* loc_network_exclude( + struct loc_network* self, struct loc_network* other); ++struct loc_network_list* loc_network_exclude_list( ++ struct loc_network* network, struct loc_network_list* list); + + // List + struct loc_network_list; +diff --git a/src/network.c b/src/network.c +index 5719111..751e8e5 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -677,6 +677,91 @@ ERROR: + return NULL; + } + ++LOC_EXPORT struct loc_network_list* loc_network_exclude_list( ++ struct loc_network* network, struct loc_network_list* list) { ++ struct loc_network_list* to_check; ++ ++ // Create a new list with all networks to look at ++ int r = loc_network_list_new(network->ctx, &to_check); ++ if (r) ++ return NULL; ++ ++ struct loc_network* subnet = NULL; ++ struct loc_network_list* subnets = NULL; ++ ++ for (unsigned int i = 0; i < loc_network_list_size(list); i++) { ++ subnet = loc_network_list_get(list, i); ++ ++ // Find all excluded networks ++ struct loc_network_list* excluded = loc_network_exclude(network, subnet); ++ if (excluded) { ++ // Add them all to the "to check" list ++ loc_network_list_merge(to_check, excluded); ++ loc_network_list_unref(excluded); ++ } ++ ++ // Cleanup ++ loc_network_unref(subnet); ++ } ++ ++ r = loc_network_list_new(network->ctx, &subnets); ++ if (r) { ++ loc_network_list_unref(to_check); ++ return NULL; ++ } ++ ++ while (!loc_network_list_empty(to_check)) { ++ struct loc_network* subnet_to_check = loc_network_list_pop(to_check); ++ ++ // Marks whether this subnet passed all checks ++ int passed = 1; ++ ++ for (unsigned int i = 0; i < loc_network_list_size(list); i++) { ++ subnet = loc_network_list_get(list, i); ++ ++ // Drop this subnet if is is already in list ++ if (loc_network_eq(subnet_to_check, subnet)) { ++ passed = 0; ++ loc_network_unref(subnet); ++ break; ++ } ++ ++ // Drop this subnet if is a subnet of another subnet ++ if (loc_network_is_subnet_of(subnet, subnet_to_check)) { ++ passed = 0; ++ loc_network_unref(subnet); ++ break; ++ } ++ ++ // Break it down if it overlaps ++ if (loc_network_overlaps(subnet_to_check, subnet)) { ++ passed = 0; ++ ++ struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet); ++ if (excluded) { ++ loc_network_list_merge(to_check, excluded); ++ loc_network_list_unref(excluded); ++ } ++ ++ loc_network_unref(subnet); ++ break; ++ } ++ ++ loc_network_unref(subnet); ++ } ++ ++ if (passed) { ++ r = loc_network_list_push(subnets, subnet_to_check); ++ } ++ ++ loc_network_unref(subnet_to_check); ++ } ++ ++ loc_network_list_unref(to_check); ++ ++ return subnets; ++} ++ + LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { + // Add country code + loc_country_code_copy(dbobj->country_code, network->country_code); +-- +2.20.1 + +From d87fd7a3d277b4b03222c7d1680e51b3e45e525b Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 20:02:03 +0000 +Subject: [PATCH 46/70] database: Add option to return networks flattened + +Signed-off-by: Michael Tremer +--- + src/database.c | 174 ++++++++++++++++++++++++++++++++++++++++--------- + 1 file changed, 143 insertions(+), 31 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 9baab33..7a3d1a7 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -118,6 +118,9 @@ struct loc_database_enumerator { + struct loc_node_stack network_stack[MAX_STACK_DEPTH]; + int network_stack_depth; + unsigned int* networks_visited; ++ ++ // For subnet search ++ struct loc_network_list* stack; + }; + + static int loc_database_read_magic(struct loc_database* db) { +@@ -935,6 +938,26 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, + + // Enumerator + ++static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { ++ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); ++ ++ // Release all references ++ loc_database_unref(enumerator->db); ++ loc_unref(enumerator->ctx); ++ ++ if (enumerator->string) ++ free(enumerator->string); ++ ++ // Free network search ++ free(enumerator->networks_visited); ++ ++ // Free subnet stack ++ if (enumerator->stack) ++ loc_network_list_unref(enumerator->stack); ++ ++ free(enumerator); ++} ++ + LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { + struct loc_database_enumerator* e = calloc(1, sizeof(*e)); +@@ -951,10 +974,16 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum + e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); + + // Initialise graph search +- //e->network_stack[++e->network_stack_depth] = 0; + e->network_stack_depth = 1; + e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited)); + ++ // Allocate stack ++ int r = loc_network_list_new(e->ctx, &e->stack); ++ if (r) { ++ loc_database_enumerator_free(e); ++ return r; ++ } ++ + DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e); + + *enumerator = e; +@@ -967,22 +996,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo + return enumerator; + } + +-static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { +- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); +- +- // Release all references +- loc_database_unref(enumerator->db); +- loc_unref(enumerator->ctx); +- +- if (enumerator->string) +- free(enumerator->string); +- +- // Free network search +- free(enumerator->networks_visited); +- +- free(enumerator); +-} +- + LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) { + if (!enumerator) + return NULL; +@@ -1116,17 +1129,13 @@ static int loc_database_enumerator_stack_push_node( + return 0; + } + +-LOC_EXPORT int loc_database_enumerator_next_network( +- struct loc_database_enumerator* enumerator, struct loc_network** network) { +- // Reset network +- *network = NULL; +- +- // Do not do anything if not in network mode +- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) ++static int __loc_database_enumerator_next_network( ++ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { ++ // Return top element from the stack ++ *network = loc_network_list_pop(enumerator->stack); ++ if (*network) + return 0; + +- int r; +- + DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", + enumerator->network_stack_depth); + +@@ -1155,7 +1164,7 @@ LOC_EXPORT int loc_database_enumerator_next_network( + enumerator->db->network_nodes_v1 + node->offset; + + // Add edges to stack +- r = loc_database_enumerator_stack_push_node(enumerator, ++ int r = loc_database_enumerator_stack_push_node(enumerator, + be32toh(n->one), 1, node->depth + 1); + + if (r) +@@ -1181,6 +1190,10 @@ LOC_EXPORT int loc_database_enumerator_next_network( + if (r) + return r; + ++ // Return all networks when the filter is disabled ++ if (!filter) ++ return 0; ++ + // Check if we are interested in this network + + // Skip if the family does not match +@@ -1223,12 +1236,111 @@ LOC_EXPORT int loc_database_enumerator_next_network( + } + + // Reached the end of the search ++ return 0; ++} + +- // Mark all nodes as non-visited +- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++) +- enumerator->networks_visited[i] = 0; ++static int __loc_database_enumerator_next_network_flattened( ++ struct loc_database_enumerator* enumerator, struct loc_network** network) { ++ // Fetch the next network ++ int r = __loc_database_enumerator_next_network(enumerator, network, 1); ++ if (r) ++ return r; + +- return 0; ++ // End if we could not read another network ++ if (!*network) ++ return 0; ++ ++ struct loc_network* subnet = NULL; ++ struct loc_network_list* subnets; ++ ++ // Create a list with all subnets ++ r = loc_network_list_new(enumerator->ctx, &subnets); ++ if (r) ++ return r; ++ ++ // Search all subnets from the database ++ while (1) { ++ // Fetch the next network in line ++ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0); ++ if (r) ++ goto END; ++ ++ // End if we did not receive another subnet ++ if (!subnet) ++ break; ++ ++ // Collect all subnets in a list ++ if (loc_network_is_subnet(*network, subnet)) { ++ r = loc_network_list_push(subnets, subnet); ++ if (r) ++ goto END; ++ ++ loc_network_unref(subnet); ++ continue; ++ } ++ ++ // If this is not a subnet, we push it back onto the stack and break ++ r = loc_network_list_push(enumerator->stack, subnet); ++ if (r) ++ goto END; ++ ++ loc_network_unref(subnet); ++ break; ++ } ++ ++ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets)); ++ ++ // We can abort here if the network has no subnets ++ if (loc_network_list_empty(subnets)) { ++ loc_network_list_unref(subnets); ++ ++ return 0; ++ } ++ ++ // If the network has any subnets, we will break it into smaller parts ++ // without the subnets. ++ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets); ++ if (!excluded || loc_network_list_empty(excluded)) { ++ r = 1; ++ goto END; ++ } ++ ++ // Sort the result ++ loc_network_list_sort(excluded); ++ ++ // Reverse the list ++ loc_network_list_reverse(excluded); ++ ++ // Replace network with the first one ++ loc_network_unref(*network); ++ ++ *network = loc_network_list_pop(excluded); ++ ++ // Push the rest onto the stack ++ loc_network_list_merge(enumerator->stack, excluded); ++ ++ loc_network_list_unref(excluded); ++ ++END: ++ if (subnet) ++ loc_network_unref(subnet); ++ ++ loc_network_list_unref(subnets); ++ ++ return r; ++} ++ ++LOC_EXPORT int loc_database_enumerator_next_network( ++ struct loc_database_enumerator* enumerator, struct loc_network** network) { ++ // Do not do anything if not in network mode ++ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) ++ return 0; ++ ++ // Flatten output? ++ if (enumerator->flatten) ++ return __loc_database_enumerator_next_network_flattened(enumerator, network); ++ ++ return __loc_database_enumerator_next_network(enumerator, network, 1); + } + + LOC_EXPORT int loc_database_enumerator_next_country( +-- +2.20.1 + +From d3ae93c27dcd7f6984fdc29cc141621e277f2e2a Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 20:09:20 +0000 +Subject: [PATCH 47/70] test: Update API + +Signed-off-by: Michael Tremer +--- + src/test-as.c | 2 +- + src/test-database.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/test-as.c b/src/test-as.c +index 839a04c..2d61675 100644 +--- a/src/test-as.c ++++ b/src/test-as.c +@@ -95,7 +95,7 @@ int main(int argc, char** argv) { + // Enumerator + + struct loc_database_enumerator* enumerator; +- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES); ++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0); + if (err) { + fprintf(stderr, "Could not create a database enumerator\n"); + exit(EXIT_FAILURE); +diff --git a/src/test-database.c b/src/test-database.c +index 4aef94e..da4b11c 100644 +--- a/src/test-database.c ++++ b/src/test-database.c +@@ -198,7 +198,7 @@ int main(int argc, char** argv) { + + // Enumerator + struct loc_database_enumerator* enumerator; +- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS); ++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0); + if (err) { + fprintf(stderr, "Could not initialise the enumerator: %d\n", err); + exit(EXIT_FAILURE); +-- +2.20.1 + +From 594ca328c6e124d0f1eb543e9c8d9bbfe8a7b628 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 12 Nov 2020 20:09:37 +0000 +Subject: [PATCH 48/70] networks: Copy all attributes when splitting networks + +Signed-off-by: Michael Tremer +--- + src/network.c | 14 ++++++++++++++ + 1 file changed, 14 insertions(+) + +diff --git a/src/network.c b/src/network.c +index 751e8e5..d67f116 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -550,6 +550,20 @@ LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* netw + if (r) + goto ERROR; + ++ // Copy country code ++ const char* country_code = loc_network_get_country_code(network); ++ if (country_code) { ++ loc_network_set_country_code(subnet1, country_code); ++ loc_network_set_country_code(subnet2, country_code); ++ } ++ ++ // Copy ASN ++ uint32_t asn = loc_network_get_asn(network); ++ if (asn) { ++ loc_network_set_asn(subnet1, asn); ++ loc_network_set_asn(subnet2, asn); ++ } ++ + loc_network_unref(subnet1); + loc_network_unref(subnet2); + +-- +2.20.1 + +From 69248038292e9ea1a4ee8912cdfc8700456753ad Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Fri, 13 Nov 2020 11:23:33 +0000 +Subject: [PATCH 49/70] database: Move network filtering into a separate + function + +Signed-off-by: Michael Tremer +--- + src/database.c | 56 +++++++++++++++++++++++--------------------------- + 1 file changed, 26 insertions(+), 30 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 7a3d1a7..72bc8eb 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1129,6 +1129,31 @@ static int loc_database_enumerator_stack_push_node( + return 0; + } + ++static int loc_database_enumerator_filter_network( ++ struct loc_database_enumerator* enumerator, struct loc_network* network) { ++ // Skip if the family does not match ++ if (enumerator->family && loc_network_address_family(network) != enumerator->family) ++ return 1; ++ ++ // Skip if the country code does not match ++ if (*enumerator->country_code && ++ !loc_network_match_country_code(network, enumerator->country_code)) ++ return 1; ++ ++ // Skip if the ASN does not match ++ if (enumerator->asn && ++ !loc_network_match_asn(network, enumerator->asn)) ++ return 1; ++ ++ // Skip if flags do not match ++ if (enumerator->flags && ++ !loc_network_match_flag(network, enumerator->flags)) ++ return 1; ++ ++ // Do not filter ++ return 0; ++} ++ + static int __loc_database_enumerator_next_network( + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { + // Return top element from the stack +@@ -1195,36 +1220,7 @@ static int __loc_database_enumerator_next_network( + return 0; + + // Check if we are interested in this network +- +- // Skip if the family does not match +- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) { +- loc_network_unref(*network); +- *network = NULL; +- +- continue; +- } +- +- // Skip if the country code does not match +- if (*enumerator->country_code && +- !loc_network_match_country_code(*network, enumerator->country_code)) { +- loc_network_unref(*network); +- *network = NULL; +- +- continue; +- } +- +- // Skip if the ASN does not match +- if (enumerator->asn && +- !loc_network_match_asn(*network, enumerator->asn)) { +- loc_network_unref(*network); +- *network = NULL; +- +- continue; +- } +- +- // Skip if flags do not match +- if (enumerator->flags && +- !loc_network_match_flag(*network, enumerator->flags)) { ++ if (loc_database_enumerator_filter_network(enumerator, *network)) { + loc_network_unref(*network); + *network = NULL; + +-- +2.20.1 + +From 2113e71bf7b997c82670c5c22cf91aa6442fe6f3 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Fri, 13 Nov 2020 11:29:02 +0000 +Subject: [PATCH 50/70] database: Filter results coming from stack + +Signed-off-by: Michael Tremer +--- + src/database.c | 18 ++++++++++++++++-- + 1 file changed, 16 insertions(+), 2 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 72bc8eb..0f3cdc2 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1157,9 +1157,23 @@ static int loc_database_enumerator_filter_network( + static int __loc_database_enumerator_next_network( + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { + // Return top element from the stack +- *network = loc_network_list_pop(enumerator->stack); +- if (*network) ++ while (1) { ++ *network = loc_network_list_pop(enumerator->stack); ++ ++ // Stack is empty ++ if (!*network) ++ break; ++ ++ // Throw away any networks by filter ++ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) { ++ loc_network_unref(*network); ++ *network = NULL; ++ continue; ++ } ++ ++ // Return result + return 0; ++ } + + DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", + enumerator->network_stack_depth); +-- +2.20.1 + +From d33753688138c9938743dafbbdddf220dd2afd14 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Fri, 13 Nov 2020 11:29:15 +0000 +Subject: [PATCH 51/70] network: Sort result of excluded lists + +Signed-off-by: Michael Tremer +--- + src/database.c | 3 --- + src/network.c | 3 +++ + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 0f3cdc2..6849d97 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1315,9 +1315,6 @@ static int __loc_database_enumerator_next_network_flattened( + goto END; + } + +- // Sort the result +- loc_network_list_sort(excluded); +- + // Reverse the list + loc_network_list_reverse(excluded); + +diff --git a/src/network.c b/src/network.c +index d67f116..9d02bf8 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -773,6 +773,9 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( + + loc_network_list_unref(to_check); + ++ // Sort the result ++ loc_network_list_sort(subnets); ++ + return subnets; + } + +-- +2.20.1 + +From 8d777f12f7ffa4df1b28d197563888296803b727 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Fri, 13 Nov 2020 11:38:15 +0000 +Subject: [PATCH 52/70] network: Add function to pop first element from stack + +Signed-off-by: Michael Tremer +--- + src/database.c | 6 ++---- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 19 +++++++++++++++++++ + 4 files changed, 23 insertions(+), 4 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 6849d97..b9d870f 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1315,15 +1315,13 @@ static int __loc_database_enumerator_next_network_flattened( + goto END; + } + +- // Reverse the list +- loc_network_list_reverse(excluded); +- + // Replace network with the first one + loc_network_unref(*network); + +- *network = loc_network_list_pop(excluded); ++ *network = loc_network_list_pop_first(excluded); + + // Push the rest onto the stack ++ loc_network_list_reverse(excluded); + loc_network_list_merge(enumerator->stack, excluded); + + loc_network_list_unref(excluded); +diff --git a/src/libloc.sym b/src/libloc.sym +index bcd11be..6139db6 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -113,6 +113,7 @@ global: + loc_network_list_merge; + loc_network_list_new; + loc_network_list_pop; ++ loc_network_list_pop_first; + loc_network_list_push; + loc_network_list_ref; + loc_network_list_reverse; +diff --git a/src/loc/network.h b/src/loc/network.h +index 40712b9..203e61c 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -76,6 +76,7 @@ void loc_network_list_dump(struct loc_network_list* list); + struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); + int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); + struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); + int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); + void loc_network_list_sort(struct loc_network_list* list); + void loc_network_list_reverse(struct loc_network_list* list); +diff --git a/src/network.c b/src/network.c +index 9d02bf8..e7dc97e 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -1231,6 +1231,25 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis + return network; + } + ++LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); ++ return NULL; ++ } ++ ++ struct loc_network* network = list->list[0]; ++ ++ // Move all elements to the top of the stack ++ for (unsigned int i = 0; i < --list->size; i++) { ++ list->list[i] = list->list[i+1]; ++ } ++ ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++ ++ return network; ++} ++ + LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { + for (unsigned int i = 0; i < list->size; i++) { + if (loc_network_eq(list->list[i], network)) +-- +2.20.1 + +From 7933f5bfb4dd7603cb646e192840762bf6394292 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Fri, 13 Nov 2020 11:43:53 +0000 +Subject: [PATCH 53/70] network: Unexport all tree functions + +These should not be exported + +Signed-off-by: Michael Tremer +--- + src/network.c | 28 ++++++++++++++-------------- + src/test-network.c | 9 +++++++++ + 2 files changed, 23 insertions(+), 14 deletions(-) + +diff --git a/src/network.c b/src/network.c +index e7dc97e..d015579 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -847,7 +847,7 @@ struct loc_network_tree_node { + struct loc_network* network; + }; + +-LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { ++int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { + struct loc_network_tree* t = calloc(1, sizeof(*t)); + if (!t) + return -ENOMEM; +@@ -867,7 +867,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree + return 0; + } + +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { ++struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { + return loc_network_tree_node_ref(tree->root); + } + +@@ -939,7 +939,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_ + return 0; + } + +-LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree, ++int loc_network_tree_walk(struct loc_network_tree* tree, + int(*filter_callback)(struct loc_network* network, void* data), + int(*callback)(struct loc_network* network, void* data), void* data) { + return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data); +@@ -954,7 +954,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) { + free(tree); + } + +-LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { ++struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { + if (--tree->refcount > 0) + return tree; + +@@ -975,13 +975,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) { + return 0; + } + +-LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) { ++int loc_network_tree_dump(struct loc_network_tree* tree) { + DEBUG(tree->ctx, "Dumping network tree at %p\n", tree); + + return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL); + } + +-LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { ++int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { + DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree); + + struct loc_network_tree_node* node = loc_network_tree_get_path(tree, +@@ -1012,7 +1012,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) { + return 0; + } + +-LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { ++size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { + size_t counter = 0; + + int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter); +@@ -1034,11 +1034,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node) + return counter; + } + +-LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { ++size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { + return __loc_network_tree_count_nodes(tree->root); + } + +-LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { ++int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { + struct loc_network_tree_node* n = calloc(1, sizeof(*n)); + if (!n) + return -ENOMEM; +@@ -1053,7 +1053,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network + return 0; + } + +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { ++struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { + if (node) + node->refcount++; + +@@ -1076,7 +1076,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) { + free(node); + } + +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { ++struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { + if (!node) + return NULL; + +@@ -1087,7 +1087,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_ + return NULL; + } + +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { ++struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { + if (index == 0) + node = node->zero; + else +@@ -1099,11 +1099,11 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne + return loc_network_tree_node_ref(node); + } + +-LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { ++int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { + return (!!node->network); + } + +-LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { ++struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { + return loc_network_ref(node->network); + } + +diff --git a/src/test-network.c b/src/test-network.c +index af1b2e6..7c90224 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -37,12 +37,14 @@ int main(int argc, char** argv) { + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); + ++#if 0 + struct loc_network_tree* tree; + err = loc_network_tree_new(ctx, &tree); + if (err) { + fprintf(stderr, "Could not create the network tree\n"); + exit(EXIT_FAILURE); + } ++#endif + + // Create a network + struct loc_network* network1; +@@ -58,12 +60,14 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++#if 0 + // Adding network to the tree + err = loc_network_tree_add_network(tree, network1); + if (err) { + fprintf(stderr, "Could not add network to the tree\n"); + exit(EXIT_FAILURE); + } ++#endif + + // Check if the first and last addresses are correct + char* string = loc_network_format_first_address(network1); +@@ -101,6 +105,7 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++#if 0 + // Adding network to the tree + err = loc_network_tree_add_network(tree, network2); + if (err) { +@@ -117,6 +122,7 @@ int main(int argc, char** argv) { + + size_t nodes = loc_network_tree_count_nodes(tree); + printf("The tree has %zu nodes\n", nodes); ++#endif + + // Check equals function + err = loc_network_eq(network1, network1); +@@ -260,7 +266,10 @@ int main(int argc, char** argv) { + loc_network_unref(network2); + loc_network_unref(network3); + loc_network_unref(network4); ++ ++#if 0 + loc_network_tree_unref(tree); ++#endif + + // And open it again from disk + struct loc_database* db; +-- +2.20.1 + +From c242f7325bd6fc4ba26047ac24196d1c161c6e01 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Fri, 13 Nov 2020 12:09:03 +0000 +Subject: [PATCH 54/70] python: Move tree flattening into C + +Signed-off-by: Michael Tremer +--- + src/python/database.c | 8 ++- + src/python/export.py | 138 +++++------------------------------------- + 2 files changed, 21 insertions(+), 125 deletions(-) + +diff --git a/src/python/database.c b/src/python/database.c +index 7f8c2c2..d169547 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -258,17 +258,19 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { + } + + static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { +- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL }; ++ char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL }; + const char* country_code = NULL; + unsigned int asn = 0; + int flags = 0; + int family = 0; ++ int flatten = 0; + +- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family)) ++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten)) + return NULL; + + struct loc_database_enumerator* enumerator; +- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0); ++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, ++ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +diff --git a/src/python/export.py b/src/python/export.py +index dd44332..be4a68e 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -87,58 +87,12 @@ class OutputWriter(object): + def _write_network(self, network): + self.f.write("%s\n" % network) + +- def write(self, network, subnets): ++ def write(self, network): + if self.flatten and self._flatten(network): + log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network)) + return + +- # Convert network into a Python object +- _network = ipaddress.ip_network("%s" % network) +- +- # Write the network when it has no subnets +- if not subnets: +- log.debug("Writing %s to %s" % (_network, self.f)) +- return self._write_network(_network) +- +- # Convert subnets into Python objects +- _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets] +- +- # Split the network into smaller bits so that +- # we can accomodate for any gaps in it later +- to_check = set() +- for _subnet in _subnets: +- to_check.update( +- _network.address_exclude(_subnet) +- ) +- +- # Clear the list of all subnets +- subnets = [] +- +- # Check if all subnets to not overlap with anything else +- while to_check: +- subnet_to_check = to_check.pop() +- +- for _subnet in _subnets: +- # Drop this subnet if it equals one of the subnets +- # or if it is subnet of one of them +- if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet): +- break +- +- # Break it down if it overlaps +- if subnet_to_check.overlaps(_subnet): +- to_check.update( +- subnet_to_check.address_exclude(_subnet) +- ) +- break +- +- # Add the subnet again as it passed the check +- else: +- subnets.append(subnet_to_check) +- +- # Write all networks as compact as possible +- for network in ipaddress.collapse_addresses(subnets): +- log.debug("Writing %s to %s" % (network, self.f)) +- self._write_network(network) ++ return self._write_network(network) + + def finish(self): + """ +@@ -188,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter): + mode = "wb" + + def _write_network(self, network): +- for address in (network.network_address, network.broadcast_address): ++ for address in (network.first_address, network.last_address): + # Convert this into a string of bits + bytes = socket.inet_pton( + socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, +@@ -231,42 +185,21 @@ class Exporter(object): + writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) + + # Get all networks that match the family +- networks = self.db.search_networks(family=family) +- +- # Create a stack with all networks in order where we can put items back +- # again and retrieve them in the next iteration. +- networks = BufferedStack(networks) ++ networks = self.db.search_networks(family=family, flatten=True) + + # Walk through all networks + for network in networks: +- # Collect all networks which are a subnet of network +- subnets = [] +- for subnet in networks: +- # If the next subnet was not a subnet, we have to push +- # it back on the stack and break this loop +- if not subnet.is_subnet_of(network): +- networks.push(subnet) +- break +- +- subnets.append(subnet) +- + # Write matching countries +- if network.country_code and network.country_code in writers: +- # Mismatching subnets +- gaps = [ +- subnet for subnet in subnets if not network.country_code == subnet.country_code +- ] +- +- writers[network.country_code].write(network, gaps) ++ try: ++ writers[network.country_code].write(network) ++ except KeyError: ++ pass + + # Write matching ASNs +- if network.asn and network.asn in writers: +- # Mismatching subnets +- gaps = [ +- subnet for subnet in subnets if not network.asn == subnet.asn +- ] +- +- writers[network.asn].write(network, gaps) ++ try: ++ writers[network.asn].write(network) ++ except KeyError: ++ pass + + # Handle flags + for flag in flags: +@@ -274,19 +207,10 @@ class Exporter(object): + # Fetch the "fake" country code + country = flags[flag] + +- if not country in writers: +- continue +- +- gaps = [ +- subnet for subnet in subnets +- if not subnet.has_flag(flag) +- ] +- +- writers[country].write(network, gaps) +- +- # Push all subnets back onto the stack +- for subnet in reversed(subnets): +- networks.push(subnet) ++ try: ++ writers[country].write(network) ++ except KeyError: ++ pass + + # Write everything to the filesystem + for writer in writers.values(): +@@ -298,33 +222,3 @@ class Exporter(object): + ) + + return os.path.join(directory, filename) +- +- +-class BufferedStack(object): +- """ +- This class takes an iterator and when being iterated +- over it returns objects from that iterator for as long +- as there are any. +- +- It additionally has a function to put an item back on +- the back so that it will be returned again at the next +- iteration. +- """ +- def __init__(self, iterator): +- self.iterator = iterator +- self.stack = [] +- +- def __iter__(self): +- return self +- +- def __next__(self): +- if self.stack: +- return self.stack.pop(0) +- +- return next(self.iterator) +- +- def push(self, elem): +- """ +- Takes an element and puts it on the stack +- """ +- self.stack.insert(0, elem) +-- +2.20.1 + +From e0b9ff5f38beb0d560b16db881647e5a75127df1 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Sun, 15 Nov 2020 15:02:28 +0000 +Subject: [PATCH 55/70] Move network lists into an own file + +Signed-off-by: Michael Tremer +--- + Makefile.am | 2 + + src/libloc.sym | 1 + + src/loc/network-list.h | 37 +++++++ + src/loc/network.h | 20 +--- + src/network-list.c | 224 +++++++++++++++++++++++++++++++++++++++++ + src/network.c | 207 +------------------------------------ + src/python/network.c | 1 + + 7 files changed, 269 insertions(+), 223 deletions(-) + create mode 100644 src/loc/network-list.h + create mode 100644 src/network-list.c + +diff --git a/Makefile.am b/Makefile.am +index a0431a6..f0d8c4c 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -96,6 +96,7 @@ pkginclude_HEADERS = \ + src/loc/database.h \ + src/loc/format.h \ + src/loc/network.h \ ++ src/loc/network-list.h \ + src/loc/private.h \ + src/loc/stringpool.h \ + src/loc/resolv.h \ +@@ -110,6 +111,7 @@ src_libloc_la_SOURCES = \ + src/country.c \ + src/database.c \ + src/network.c \ ++ src/network-list.c \ + src/resolv.c \ + src/stringpool.c \ + src/writer.c +diff --git a/src/libloc.sym b/src/libloc.sym +index 6139db6..453a1be 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -87,6 +87,7 @@ global: + loc_network_format_last_address; + loc_network_get_asn; + loc_network_get_country_code; ++ loc_network_gt; + loc_network_has_flag; + loc_network_is_subnet; + loc_network_is_subnet_of; +diff --git a/src/loc/network-list.h b/src/loc/network-list.h +new file mode 100644 +index 0000000..af3b28d +--- /dev/null ++++ b/src/loc/network-list.h +@@ -0,0 +1,37 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2020 IPFire Development Team ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#ifndef LIBLOC_NETWORK_LIST_H ++#define LIBLOC_NETWORK_LIST_H ++ ++struct loc_network_list; ++int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); ++struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); ++struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); ++size_t loc_network_list_size(struct loc_network_list* list); ++int loc_network_list_empty(struct loc_network_list* list); ++void loc_network_list_clear(struct loc_network_list* list); ++void loc_network_list_dump(struct loc_network_list* list); ++struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); ++int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); ++struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); ++int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); ++void loc_network_list_sort(struct loc_network_list* list); ++void loc_network_list_reverse(struct loc_network_list* list); ++int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); ++ ++#endif +diff --git a/src/loc/network.h b/src/loc/network.h +index 203e61c..d86b685 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -21,6 +21,7 @@ + + #include + #include ++#include + + enum loc_network_flags { + LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1 +@@ -55,6 +56,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); + int loc_network_match_flag(struct loc_network* network, uint32_t flag); + + int loc_network_eq(struct loc_network* self, struct loc_network* other); ++int loc_network_gt(struct loc_network* self, struct loc_network* other); + int loc_network_overlaps(struct loc_network* self, struct loc_network* other); + int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); + int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); +@@ -64,24 +66,6 @@ struct loc_network_list* loc_network_exclude( + struct loc_network_list* loc_network_exclude_list( + struct loc_network* network, struct loc_network_list* list); + +-// List +-struct loc_network_list; +-int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); +-struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); +-struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); +-size_t loc_network_list_size(struct loc_network_list* list); +-int loc_network_list_empty(struct loc_network_list* list); +-void loc_network_list_clear(struct loc_network_list* list); +-void loc_network_list_dump(struct loc_network_list* list); +-struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); +-int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); +-struct loc_network* loc_network_list_pop(struct loc_network_list* list); +-struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); +-int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); +-void loc_network_list_sort(struct loc_network_list* list); +-void loc_network_list_reverse(struct loc_network_list* list); +-int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); +- + #ifdef LIBLOC_PRIVATE + + int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj); +diff --git a/src/network-list.c b/src/network-list.c +new file mode 100644 +index 0000000..1f6e80e +--- /dev/null ++++ b/src/network-list.c +@@ -0,0 +1,224 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2020 IPFire Development Team ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#include ++#include ++ ++#include ++#include ++#include ++ ++struct loc_network_list { ++ struct loc_ctx* ctx; ++ int refcount; ++ ++ struct loc_network* list[1024]; ++ size_t size; ++ size_t max_size; ++}; ++ ++LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, ++ struct loc_network_list** list) { ++ struct loc_network_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; ++ ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; ++ ++ // Do not allow this list to grow larger than this ++ l->max_size = 1024; ++ ++ DEBUG(l->ctx, "Network list allocated at %p\n", l); ++ *list = l; ++ return 0; ++} ++ ++LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { ++ list->refcount++; ++ ++ return list; ++} ++ ++static void loc_network_list_free(struct loc_network_list* list) { ++ DEBUG(list->ctx, "Releasing network list at %p\n", list); ++ ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_network_unref(list->list[i]); ++ ++ loc_unref(list->ctx); ++ free(list); ++} ++ ++LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { ++ if (!list) ++ return NULL; ++ ++ if (--list->refcount > 0) ++ return list; ++ ++ loc_network_list_free(list); ++ return NULL; ++} ++ ++LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { ++ return list->size; ++} ++ ++LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { ++ return list->size == 0; ++} ++ ++LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_network_unref(list->list[i]); ++ ++ list->size = 0; ++} ++ ++LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { ++ struct loc_network* network; ++ char* s; ++ ++ for (unsigned int i = 0; i < list->size; i++) { ++ network = list->list[i]; ++ ++ s = loc_network_str(network); ++ ++ INFO(list->ctx, "%s\n", s); ++ free(s); ++ } ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; ++ ++ return loc_network_ref(list->list[index]); ++} ++ ++LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { ++ // Do not add networks that are already on the list ++ if (loc_network_list_contains(list, network)) ++ return 0; ++ ++ // Check if we have space left ++ if (list->size == list->max_size) { ++ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); ++ return -ENOMEM; ++ } ++ ++ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); ++ ++ list->list[list->size++] = loc_network_ref(network); ++ ++ return 0; ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); ++ return NULL; ++ } ++ ++ struct loc_network* network = list->list[--list->size]; ++ ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++ ++ return network; ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); ++ return NULL; ++ } ++ ++ struct loc_network* network = list->list[0]; ++ ++ // Move all elements to the top of the stack ++ for (unsigned int i = 0; i < --list->size; i++) { ++ list->list[i] = list->list[i+1]; ++ } ++ ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++ ++ return network; ++} ++ ++LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_network_eq(list->list[i], network)) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { ++ // Do nothing for invalid indices ++ if (i1 >= list->size || i2 >= list->size) ++ return; ++ ++ struct loc_network* network1 = list->list[i1]; ++ struct loc_network* network2 = list->list[i2]; ++ ++ list->list[i1] = network2; ++ list->list[i2] = network1; ++} ++ ++LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { ++ unsigned int i = 0; ++ unsigned int j = list->size - 1; ++ ++ while (i < j) { ++ loc_network_list_swap(list, i++, j--); ++ } ++} ++ ++LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { ++ unsigned int n = list->size; ++ int swapped; ++ ++ do { ++ swapped = 0; ++ ++ for (unsigned int i = 1; i < n; i++) { ++ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { ++ loc_network_list_swap(list, i-1, i); ++ swapped = 1; ++ } ++ } ++ ++ n--; ++ } while (swapped); ++} ++ ++LOC_EXPORT int loc_network_list_merge( ++ struct loc_network_list* self, struct loc_network_list* other) { ++ int r; ++ ++ for (unsigned int i = 0; i < other->size; i++) { ++ r = loc_network_list_push(self, other->list[i]); ++ if (r) ++ return r; ++ } ++ ++ return 0; ++} +diff --git a/src/network.c b/src/network.c +index d015579..28ca2df 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -29,6 +29,7 @@ + #include + #include + #include ++#include + #include + + struct loc_network { +@@ -436,7 +437,7 @@ LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* othe + return 1; + } + +-static int loc_network_gt(struct loc_network* self, struct loc_network* other) { ++LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) { + // Families must match + if (self->family != other->family) + return -1; +@@ -1106,207 +1107,3 @@ int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { + struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { + return loc_network_ref(node->network); + } +- +-// List +- +-struct loc_network_list { +- struct loc_ctx* ctx; +- int refcount; +- +- struct loc_network* list[1024]; +- size_t size; +- size_t max_size; +-}; +- +-LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, +- struct loc_network_list** list) { +- struct loc_network_list* l = calloc(1, sizeof(*l)); +- if (!l) +- return -ENOMEM; +- +- l->ctx = loc_ref(ctx); +- l->refcount = 1; +- +- // Do not allow this list to grow larger than this +- l->max_size = 1024; +- +- DEBUG(l->ctx, "Network list allocated at %p\n", l); +- *list = l; +- return 0; +-} +- +-LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { +- list->refcount++; +- +- return list; +-} +- +-static void loc_network_list_free(struct loc_network_list* list) { +- DEBUG(list->ctx, "Releasing network list at %p\n", list); +- +- for (unsigned int i = 0; i < list->size; i++) +- loc_network_unref(list->list[i]); +- +- loc_unref(list->ctx); +- free(list); +-} +- +-LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { +- if (!list) +- return NULL; +- +- if (--list->refcount > 0) +- return list; +- +- loc_network_list_free(list); +- return NULL; +-} +- +-LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { +- return list->size; +-} +- +-LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { +- return list->size == 0; +-} +- +-LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { +- for (unsigned int i = 0; i < list->size; i++) +- loc_network_unref(list->list[i]); +- +- list->size = 0; +-} +- +-LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { +- struct loc_network* network; +- char* s; +- +- for (unsigned int i = 0; i < list->size; i++) { +- network = list->list[i]; +- +- s = loc_network_str(network); +- +- INFO(list->ctx, "%s\n", s); +- free(s); +- } +-} +- +-LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { +- // Check index +- if (index >= list->size) +- return NULL; +- +- return loc_network_ref(list->list[index]); +-} +- +-LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { +- // Do not add networks that are already on the list +- if (loc_network_list_contains(list, network)) +- return 0; +- +- // Check if we have space left +- if (list->size == list->max_size) { +- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); +- return -ENOMEM; +- } +- +- DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); +- +- list->list[list->size++] = loc_network_ref(network); +- +- return 0; +-} +- +-LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { +- // Return nothing when empty +- if (loc_network_list_empty(list)) { +- DEBUG(list->ctx, "%p: Popped empty stack\n", list); +- return NULL; +- } +- +- struct loc_network* network = list->list[--list->size]; +- +- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); +- +- return network; +-} +- +-LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { +- // Return nothing when empty +- if (loc_network_list_empty(list)) { +- DEBUG(list->ctx, "%p: Popped empty stack\n", list); +- return NULL; +- } +- +- struct loc_network* network = list->list[0]; +- +- // Move all elements to the top of the stack +- for (unsigned int i = 0; i < --list->size; i++) { +- list->list[i] = list->list[i+1]; +- } +- +- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); +- +- return network; +-} +- +-LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { +- for (unsigned int i = 0; i < list->size; i++) { +- if (loc_network_eq(list->list[i], network)) +- return 1; +- } +- +- return 0; +-} +- +-static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { +- // Do nothing for invalid indices +- if (i1 >= list->size || i2 >= list->size) +- return; +- +- struct loc_network* network1 = list->list[i1]; +- struct loc_network* network2 = list->list[i2]; +- +- list->list[i1] = network2; +- list->list[i2] = network1; +-} +- +-LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { +- unsigned int i = 0; +- unsigned int j = list->size - 1; +- +- while (i < j) { +- loc_network_list_swap(list, i++, j--); +- } +-} +- +-LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { +- unsigned int n = list->size; +- int swapped; +- +- do { +- swapped = 0; +- +- for (unsigned int i = 1; i < n; i++) { +- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { +- loc_network_list_swap(list, i-1, i); +- swapped = 1; +- } +- } +- +- n--; +- } while (swapped); +-} +- +-LOC_EXPORT int loc_network_list_merge( +- struct loc_network_list* self, struct loc_network_list* other) { +- int r; +- +- for (unsigned int i = 0; i < other->size; i++) { +- r = loc_network_list_push(self, other->list[i]); +- if (r) +- return r; +- } +- +- return 0; +-} +diff --git a/src/python/network.c b/src/python/network.c +index 11f672b..ed91d65 100644 +--- a/src/python/network.c ++++ b/src/python/network.c +@@ -20,6 +20,7 @@ + + #include + #include ++#include + + #include "locationmodule.h" + #include "network.h" +-- +2.20.1 + +From e646a8f35ec7eff009414b3fd107c9af5cf39a86 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Mon, 16 Nov 2020 15:13:28 +0000 +Subject: [PATCH 56/70] Implement filtering for multiple countries in the + enumerator + +This will allow us to speed up the export of the database +if only a few countries should be returned. + +Signed-off-by: Michael Tremer +--- + Makefile.am | 2 + + src/country-list.c | 138 +++++++++++++++++++++++++++++++++++++++++ + src/country.c | 3 + + src/database.c | 47 ++++++-------- + src/libloc.sym | 15 ++++- + src/loc/country-list.h | 43 +++++++++++++ + src/loc/database.h | 5 +- + src/python/database.c | 57 ++++++++++++++--- + 8 files changed, 274 insertions(+), 36 deletions(-) + create mode 100644 src/country-list.c + create mode 100644 src/loc/country-list.h + +diff --git a/Makefile.am b/Makefile.am +index f0d8c4c..f4ca3c8 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -93,6 +93,7 @@ pkginclude_HEADERS = \ + src/loc/as.h \ + src/loc/compat.h \ + src/loc/country.h \ ++ src/loc/country-list.h \ + src/loc/database.h \ + src/loc/format.h \ + src/loc/network.h \ +@@ -109,6 +110,7 @@ src_libloc_la_SOURCES = \ + src/libloc.c \ + src/as.c \ + src/country.c \ ++ src/country-list.c \ + src/database.c \ + src/network.c \ + src/network-list.c \ +diff --git a/src/country-list.c b/src/country-list.c +new file mode 100644 +index 0000000..ae0d71a +--- /dev/null ++++ b/src/country-list.c +@@ -0,0 +1,138 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2020 IPFire Development Team ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#include ++#include ++ ++#include ++#include ++#include ++ ++struct loc_country_list { ++ struct loc_ctx* ctx; ++ int refcount; ++ ++ struct loc_country* list[1024]; ++ size_t size; ++ size_t max_size; ++}; ++ ++LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, ++ struct loc_country_list** list) { ++ struct loc_country_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; ++ ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; ++ ++ // Do not allow this list to grow larger than this ++ l->max_size = 1024; ++ ++ DEBUG(l->ctx, "Country list allocated at %p\n", l); ++ *list = l; ++ ++ return 0; ++} ++ ++LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) { ++ list->refcount++; ++ ++ return list; ++} ++ ++static void loc_country_list_free(struct loc_country_list* list) { ++ DEBUG(list->ctx, "Releasing country list at %p\n", list); ++ ++ loc_country_list_clear(list); ++ ++ loc_unref(list->ctx); ++ free(list); ++} ++ ++LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) { ++ if (!list) ++ return NULL; ++ ++ if (--list->refcount > 0) ++ return list; ++ ++ loc_country_list_free(list); ++ return NULL; ++} ++ ++LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) { ++ return list->size; ++} ++ ++LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { ++ return list->size == 0; ++} ++ ++LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_country_unref(list->list[i]); ++} ++ ++LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; ++ ++ return loc_country_ref(list->list[index]); ++} ++ ++LOC_EXPORT int loc_country_list_append( ++ struct loc_country_list* list, struct loc_country* country) { ++ if (loc_country_list_contains(list, country)) ++ return 0; ++ ++ // Check if we have space left ++ if (list->size == list->max_size) { ++ ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list); ++ return -ENOMEM; ++ } ++ ++ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); ++ ++ list->list[list->size++] = loc_country_ref(country); ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_country_list_contains( ++ struct loc_country_list* list, struct loc_country* country) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_country_cmp(country, list->list[i]) == 0) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_country_list_contains_code( ++ struct loc_country_list* list, const char* code) { ++ struct loc_country* country; ++ ++ int r = loc_country_new(list->ctx, &country, code); ++ if (r) ++ return -1; ++ ++ r = loc_country_list_contains(list, country); ++ loc_country_unref(country); ++ ++ return r; ++} +diff --git a/src/country.c b/src/country.c +index 2ba93e6..7aac0db 100644 +--- a/src/country.c ++++ b/src/country.c +@@ -34,6 +34,9 @@ struct loc_country { + }; + + LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) { ++ if (!loc_country_code_is_valid(country_code)) ++ return -EINVAL; ++ + struct loc_country* c = calloc(1, sizeof(*c)); + if (!c) + return -ENOMEM; +diff --git a/src/database.c b/src/database.c +index b9d870f..29823b2 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -40,6 +40,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -99,7 +100,7 @@ struct loc_database_enumerator { + + // Search string + char* string; +- char country_code[3]; ++ struct loc_country_list* countries; + uint32_t asn; + enum loc_network_flags flags; + int family; +@@ -1017,33 +1018,20 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator + return 0; + } + +-LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) { +- // Set empty country code +- if (!country_code || !*country_code) { +- *enumerator->country_code = '\0'; +- return 0; +- } ++LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries( ++ struct loc_database_enumerator* enumerator) { ++ if (!enumerator->countries) ++ return NULL; + +- // Treat A1, A2, A3 as special country codes, +- // but perform search for flags instead +- if (strcmp(country_code, "A1") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_ANONYMOUS_PROXY); +- } else if (strcmp(country_code, "A2") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_SATELLITE_PROVIDER); +- } else if (strcmp(country_code, "A3") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_ANYCAST); +- } ++ return loc_country_list_ref(enumerator->countries); ++} + +- // Country codes must be two characters +- if (!loc_country_code_is_valid(country_code)) +- return -EINVAL; ++LOC_EXPORT int loc_database_enumerator_set_countries( ++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) { ++ if (enumerator->countries) ++ loc_country_list_unref(enumerator->countries); + +- for (unsigned int i = 0; i < 3; i++) { +- enumerator->country_code[i] = country_code[i]; +- } ++ enumerator->countries = loc_country_list_ref(countries); + + return 0; + } +@@ -1129,6 +1117,12 @@ static int loc_database_enumerator_stack_push_node( + return 0; + } + ++static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) { ++ const char* country_code = loc_network_get_country_code(network); ++ ++ return loc_country_list_contains_code(countries, country_code); ++} ++ + static int loc_database_enumerator_filter_network( + struct loc_database_enumerator* enumerator, struct loc_network* network) { + // Skip if the family does not match +@@ -1136,8 +1130,7 @@ static int loc_database_enumerator_filter_network( + return 1; + + // Skip if the country code does not match +- if (*enumerator->country_code && +- !loc_network_match_country_code(network, enumerator->country_code)) ++ if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries)) + return 1; + + // Skip if the ASN does not match +diff --git a/src/libloc.sym b/src/libloc.sym +index 453a1be..40e9f88 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -49,6 +49,18 @@ global: + loc_country_set_name; + loc_country_unref; + ++ # Country List ++ loc_country_list_append; ++ loc_country_list_clear; ++ loc_country_list_contains; ++ loc_country_list_contains_code; ++ loc_country_list_empty; ++ loc_country_list_get; ++ loc_country_list_new; ++ loc_country_list_ref; ++ loc_country_list_size; ++ loc_country_list_unref; ++ + # Database + loc_database_add_as; + loc_database_count_as; +@@ -66,13 +78,14 @@ global: + loc_database_verify; + + # Database Enumerator ++ loc_database_enumerator_get_countries; + loc_database_enumerator_new; + loc_database_enumerator_next_as; + loc_database_enumerator_next_country; + loc_database_enumerator_next_network; + loc_database_enumerator_ref; + loc_database_enumerator_set_asn; +- loc_database_enumerator_set_country_code; ++ loc_database_enumerator_set_countries; + loc_database_enumerator_set_family; + loc_database_enumerator_set_flag; + loc_database_enumerator_set_string; +diff --git a/src/loc/country-list.h b/src/loc/country-list.h +new file mode 100644 +index 0000000..a7f818a +--- /dev/null ++++ b/src/loc/country-list.h +@@ -0,0 +1,43 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2017 IPFire Development Team ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#ifndef LIBLOC_COUNTRY_LIST_H ++#define LIBLOC_COUNTRY_LIST_H ++ ++#include ++ ++#include ++#include ++ ++struct loc_country_list; ++ ++int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list); ++struct loc_country_list* loc_country_list_ref(struct loc_country_list* list); ++struct loc_country_list* loc_country_list_unref(struct loc_country_list* list); ++ ++size_t loc_country_list_size(struct loc_country_list* list); ++int loc_country_list_empty(struct loc_country_list* list); ++void loc_country_list_clear(struct loc_country_list* list); ++ ++struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index); ++int loc_country_list_append(struct loc_country_list* list, struct loc_country* country); ++ ++int loc_country_list_contains( ++ struct loc_country_list* list, struct loc_country* country); ++int loc_country_list_contains_code( ++ struct loc_country_list* list, const char* code); ++ ++#endif +diff --git a/src/loc/database.h b/src/loc/database.h +index 14eb5ea..246e5c5 100644 +--- a/src/loc/database.h ++++ b/src/loc/database.h +@@ -25,6 +25,7 @@ + #include + #include + #include ++#include + + struct loc_database; + int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f); +@@ -66,7 +67,9 @@ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_ + struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); + + int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string); +-int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code); ++struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); ++int loc_database_enumerator_set_countries( ++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries); + int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn); + int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); + int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); +diff --git a/src/python/database.c b/src/python/database.c +index d169547..e6f6f37 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -258,14 +258,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { + } + + static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { +- char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL }; +- const char* country_code = NULL; ++ char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL }; ++ PyObject* country_codes = NULL; + unsigned int asn = 0; + int flags = 0; + int family = 0; + int flatten = 0; + +- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten)) ++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist, ++ &PyList_Type, &country_codes, &asn, &flags, &family, &flatten)) + return NULL; + + struct loc_database_enumerator* enumerator; +@@ -277,13 +278,55 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, + } + + // Set country code we are searching for +- if (country_code) { +- r = loc_database_enumerator_set_country_code(enumerator, country_code); +- ++ if (country_codes) { ++ struct loc_country_list* countries; ++ r = loc_country_list_new(loc_ctx, &countries); + if (r) { +- PyErr_SetFromErrno(PyExc_SystemError); ++ PyErr_SetString(PyExc_SystemError, "Could not create country list"); + return NULL; + } ++ ++ for (unsigned int i = 0; i < PyList_Size(country_codes); i++) { ++ PyObject* item = PyList_GetItem(country_codes, i); ++ ++ if (!PyUnicode_Check(item)) { ++ PyErr_SetString(PyExc_TypeError, "Country codes must be strings"); ++ loc_country_list_unref(countries); ++ return NULL; ++ } ++ ++ const char* country_code = PyUnicode_AsUTF8(item); ++ ++ struct loc_country* country; ++ r = loc_country_new(loc_ctx, &country, country_code); ++ if (r) { ++ if (r == -EINVAL) { ++ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code); ++ } else { ++ PyErr_SetString(PyExc_SystemError, "Could not create country"); ++ } ++ ++ loc_country_list_unref(countries); ++ return NULL; ++ } ++ ++ // Append it to the list ++ r = loc_country_list_append(countries, country); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not append country to the list"); ++ ++ loc_country_list_unref(countries); ++ loc_country_unref(country); ++ return NULL; ++ } ++ ++ loc_country_unref(country); ++ } ++ ++ loc_database_enumerator_set_countries(enumerator, countries); ++ ++ Py_DECREF(country_codes); ++ loc_country_list_unref(countries); + } + + // Set the ASN we are searching for +-- +2.20.1 + +From 7af51f8a579c79714992a3e175036fb511139310 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Mon, 16 Nov 2020 15:20:50 +0000 +Subject: [PATCH 57/70] python: Only return country codes we want + +Signed-off-by: Michael Tremer +--- + src/python/export.py | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/src/python/export.py b/src/python/export.py +index be4a68e..5e7fe53 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -184,8 +184,14 @@ class Exporter(object): + + writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) + ++ # Filter countries from special country codes ++ country_codes = [ ++ country_code for country_code in countries if not country_code in flags.values() ++ ] ++ + # Get all networks that match the family +- networks = self.db.search_networks(family=family, flatten=True) ++ networks = self.db.search_networks(family=family, ++ country_codes=country_codes, flatten=True) + + # Walk through all networks + for network in networks: +-- +2.20.1 + +From bd1dc6bf6fe4ce40bf12e7426e283b31afd274e1 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Mon, 16 Nov 2020 15:25:15 +0000 +Subject: [PATCH 58/70] database: Filter flags in C + +Signed-off-by: Michael Tremer +--- + src/python/export.py | 16 +++++++++++----- + 1 file changed, 11 insertions(+), 5 deletions(-) + +diff --git a/src/python/export.py b/src/python/export.py +index 5e7fe53..739742f 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -29,7 +29,7 @@ import _location + log = logging.getLogger("location.export") + log.propagate = 1 + +-flags = { ++FLAGS = { + _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", + _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", + _location.NETWORK_FLAG_ANYCAST : "A3", +@@ -186,12 +186,18 @@ class Exporter(object): + + # Filter countries from special country codes + country_codes = [ +- country_code for country_code in countries if not country_code in flags.values() ++ country_code for country_code in countries if not country_code in FLAGS.values() + ] + ++ # Collect flags ++ flags = 0 ++ for flag in FLAGS: ++ if FLAGS[flag] in countries: ++ flags |= flag ++ + # Get all networks that match the family + networks = self.db.search_networks(family=family, +- country_codes=country_codes, flatten=True) ++ country_codes=country_codes, flags=flags, flatten=True) + + # Walk through all networks + for network in networks: +@@ -208,10 +214,10 @@ class Exporter(object): + pass + + # Handle flags +- for flag in flags: ++ for flag in FLAGS: + if network.has_flag(flag): + # Fetch the "fake" country code +- country = flags[flag] ++ country = FLAGS[flag] + + try: + writers[country].write(network) +-- +2.20.1 + +From 84a2f0c2d9cbf8ae4225802c29ccba86561c77ed Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 17 Nov 2020 16:46:48 +0000 +Subject: [PATCH 59/70] as: Add list for easier processing + +Signed-off-by: Michael Tremer +--- + Makefile.am | 2 + + src/as-list.c | 138 ++++++++++++++++++++++++++++++++++++++++++ + src/database.c | 29 +++++++-- + src/libloc.sym | 15 ++++- + src/loc/as-list.h | 41 +++++++++++++ + src/loc/database.h | 5 +- + src/python/database.c | 58 ++++++++++++++++-- + src/python/export.py | 2 +- + 8 files changed, 275 insertions(+), 15 deletions(-) + create mode 100644 src/as-list.c + create mode 100644 src/loc/as-list.h + +diff --git a/Makefile.am b/Makefile.am +index f4ca3c8..d0cc793 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -91,6 +91,7 @@ EXTRA_DIST += \ + pkginclude_HEADERS = \ + src/loc/libloc.h \ + src/loc/as.h \ ++ src/loc/as-list.h \ + src/loc/compat.h \ + src/loc/country.h \ + src/loc/country-list.h \ +@@ -109,6 +110,7 @@ lib_LTLIBRARIES = \ + src_libloc_la_SOURCES = \ + src/libloc.c \ + src/as.c \ ++ src/as-list.c \ + src/country.c \ + src/country-list.c \ + src/database.c \ +diff --git a/src/as-list.c b/src/as-list.c +new file mode 100644 +index 0000000..7c69eb0 +--- /dev/null ++++ b/src/as-list.c +@@ -0,0 +1,138 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2020 IPFire Development Team ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#include ++#include ++ ++#include ++#include ++#include ++ ++struct loc_as_list { ++ struct loc_ctx* ctx; ++ int refcount; ++ ++ struct loc_as* list[1024]; ++ size_t size; ++ size_t max_size; ++}; ++ ++LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, ++ struct loc_as_list** list) { ++ struct loc_as_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; ++ ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; ++ ++ // Do not allow this list to grow larger than this ++ l->max_size = 1024; ++ ++ DEBUG(l->ctx, "AS list allocated at %p\n", l); ++ *list = l; ++ ++ return 0; ++} ++ ++LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) { ++ list->refcount++; ++ ++ return list; ++} ++ ++static void loc_as_list_free(struct loc_as_list* list) { ++ DEBUG(list->ctx, "Releasing AS list at %p\n", list); ++ ++ loc_as_list_clear(list); ++ ++ loc_unref(list->ctx); ++ free(list); ++} ++ ++LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) { ++ if (!list) ++ return NULL; ++ ++ if (--list->refcount > 0) ++ return list; ++ ++ loc_as_list_free(list); ++ return NULL; ++} ++ ++LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) { ++ return list->size; ++} ++ ++LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { ++ return list->size == 0; ++} ++ ++LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_as_unref(list->list[i]); ++} ++ ++LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; ++ ++ return loc_as_ref(list->list[index]); ++} ++ ++LOC_EXPORT int loc_as_list_append( ++ struct loc_as_list* list, struct loc_as* as) { ++ if (loc_as_list_contains(list, as)) ++ return 0; ++ ++ // Check if we have space left ++ if (list->size == list->max_size) { ++ ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list); ++ return -ENOMEM; ++ } ++ ++ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); ++ ++ list->list[list->size++] = loc_as_ref(as); ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_as_list_contains( ++ struct loc_as_list* list, struct loc_as* as) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_as_cmp(as, list->list[i]) == 0) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_as_list_contains_number( ++ struct loc_as_list* list, uint32_t number) { ++ struct loc_as* as; ++ ++ int r = loc_as_new(list->ctx, &as, number); ++ if (r) ++ return -1; ++ ++ r = loc_as_list_contains(list, as); ++ loc_as_unref(as); ++ ++ return r; ++} +diff --git a/src/database.c b/src/database.c +index 29823b2..51cb5cd 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -38,6 +38,7 @@ + + #include + #include ++#include + #include + #include + #include +@@ -101,7 +102,7 @@ struct loc_database_enumerator { + // Search string + char* string; + struct loc_country_list* countries; +- uint32_t asn; ++ struct loc_as_list* asns; + enum loc_network_flags flags; + int family; + +@@ -1036,9 +1037,20 @@ LOC_EXPORT int loc_database_enumerator_set_countries( + return 0; + } + +-LOC_EXPORT int loc_database_enumerator_set_asn( +- struct loc_database_enumerator* enumerator, unsigned int asn) { +- enumerator->asn = asn; ++LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns( ++ struct loc_database_enumerator* enumerator) { ++ if (!enumerator->asns) ++ return NULL; ++ ++ return loc_as_list_ref(enumerator->asns); ++} ++ ++LOC_EXPORT int loc_database_enumerator_set_asns( ++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) { ++ if (enumerator->asns) ++ loc_as_list_unref(enumerator->asns); ++ ++ enumerator->asns = loc_as_list_ref(asns); + + return 0; + } +@@ -1123,6 +1135,12 @@ static int loc_network_match_countries(struct loc_network* network, struct loc_c + return loc_country_list_contains_code(countries, country_code); + } + ++static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { ++ uint32_t asn = loc_network_get_asn(network); ++ ++ return loc_as_list_contains_number(asns, asn); ++} ++ + static int loc_database_enumerator_filter_network( + struct loc_database_enumerator* enumerator, struct loc_network* network) { + // Skip if the family does not match +@@ -1134,8 +1152,7 @@ static int loc_database_enumerator_filter_network( + return 1; + + // Skip if the ASN does not match +- if (enumerator->asn && +- !loc_network_match_asn(network, enumerator->asn)) ++ if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) + return 1; + + // Skip if flags do not match +diff --git a/src/libloc.sym b/src/libloc.sym +index 40e9f88..53273cd 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -37,6 +37,18 @@ global: + loc_as_set_name; + loc_as_unref; + ++ # AS List ++ loc_as_list_append; ++ loc_as_list_clear; ++ loc_as_list_contains; ++ loc_as_list_contains_number; ++ loc_as_list_empty; ++ loc_as_list_get; ++ loc_as_list_new; ++ loc_as_list_ref; ++ loc_as_list_size; ++ loc_as_list_unref; ++ + # Country + loc_country_cmp; + loc_country_code_is_valid; +@@ -78,13 +90,14 @@ global: + loc_database_verify; + + # Database Enumerator ++ loc_database_enumerator_get_asns; + loc_database_enumerator_get_countries; + loc_database_enumerator_new; + loc_database_enumerator_next_as; + loc_database_enumerator_next_country; + loc_database_enumerator_next_network; + loc_database_enumerator_ref; +- loc_database_enumerator_set_asn; ++ loc_database_enumerator_set_asns; + loc_database_enumerator_set_countries; + loc_database_enumerator_set_family; + loc_database_enumerator_set_flag; +diff --git a/src/loc/as-list.h b/src/loc/as-list.h +new file mode 100644 +index 0000000..7b5c4e8 +--- /dev/null ++++ b/src/loc/as-list.h +@@ -0,0 +1,41 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2017 IPFire Development Team ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#ifndef LIBLOC_AS_LIST_H ++#define LIBLOC_AS_LIST_H ++ ++#include ++#include ++ ++struct loc_as_list; ++ ++int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list); ++struct loc_as_list* loc_as_list_ref(struct loc_as_list* list); ++struct loc_as_list* loc_as_list_unref(struct loc_as_list* list); ++ ++size_t loc_as_list_size(struct loc_as_list* list); ++int loc_as_list_empty(struct loc_as_list* list); ++void loc_as_list_clear(struct loc_as_list* list); ++ ++struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index); ++int loc_as_list_append(struct loc_as_list* list, struct loc_as* as); ++ ++int loc_as_list_contains( ++ struct loc_as_list* list, struct loc_as* as); ++int loc_as_list_contains_number( ++ struct loc_as_list* list, uint32_t number); ++ ++#endif +diff --git a/src/loc/database.h b/src/loc/database.h +index 246e5c5..70801f0 100644 +--- a/src/loc/database.h ++++ b/src/loc/database.h +@@ -70,7 +70,10 @@ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerato + struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); + int loc_database_enumerator_set_countries( + struct loc_database_enumerator* enumerator, struct loc_country_list* countries); +-int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn); ++struct loc_as_list* loc_database_enumerator_get_asns( ++ struct loc_database_enumerator* enumerator); ++int loc_database_enumerator_set_asns( ++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns); + int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); + int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); + int loc_database_enumerator_next_as( +diff --git a/src/python/database.c b/src/python/database.c +index e6f6f37..38a804c 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -17,6 +17,8 @@ + #include + + #include ++#include ++#include + #include + + #include "locationmodule.h" +@@ -258,15 +260,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { + } + + static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { +- char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL }; ++ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL }; + PyObject* country_codes = NULL; +- unsigned int asn = 0; ++ PyObject* asn_list = NULL; + int flags = 0; + int family = 0; + int flatten = 0; + +- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist, +- &PyList_Type, &country_codes, &asn, &flags, &family, &flatten)) ++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist, ++ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten)) + return NULL; + + struct loc_database_enumerator* enumerator; +@@ -330,13 +332,57 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, + } + + // Set the ASN we are searching for +- if (asn) { +- r = loc_database_enumerator_set_asn(enumerator, asn); ++ if (asn_list) { ++ struct loc_as_list* asns; ++ r = loc_as_list_new(loc_ctx, &asns); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not create AS list"); ++ return NULL; ++ } ++ ++ for (unsigned int i = 0; i < PyList_Size(asn_list); i++) { ++ PyObject* item = PyList_GetItem(asn_list, i); ++ ++ if (!PyLong_Check(item)) { ++ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers"); ++ ++ loc_as_list_unref(asns); ++ return NULL; ++ } ++ ++ unsigned long number = PyLong_AsLong(item); + ++ struct loc_as* as; ++ r = loc_as_new(loc_ctx, &as, number); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not create AS"); ++ ++ loc_as_list_unref(asns); ++ loc_as_unref(as); ++ return NULL; ++ } ++ ++ r = loc_as_list_append(asns, as); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list"); ++ ++ loc_as_list_unref(asns); ++ loc_as_unref(as); ++ return NULL; ++ } ++ ++ loc_as_unref(as); ++ } ++ ++ r = loc_database_enumerator_set_asns(enumerator, asns); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); ++ ++ loc_as_list_unref(asns); + return NULL; + } ++ ++ loc_as_list_unref(asns); + } + + // Set the flags we are searching for +diff --git a/src/python/export.py b/src/python/export.py +index 739742f..f675eb3 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -197,7 +197,7 @@ class Exporter(object): + + # Get all networks that match the family + networks = self.db.search_networks(family=family, +- country_codes=country_codes, flags=flags, flatten=True) ++ country_codes=country_codes, asns=asns, flags=flags, flatten=True) + + # Walk through all networks + for network in networks: +-- +2.20.1 + +From 50120b991fc2fa4b7813096de87b42d700faf3e6 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 17 Nov 2020 16:56:43 +0000 +Subject: [PATCH 60/70] database: Simplify network matching code + +Signed-off-by: Michael Tremer +--- + src/database.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 51cb5cd..1a354f6 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node( + return 0; + } + +-static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) { +- const char* country_code = loc_network_get_country_code(network); +- +- return loc_country_list_contains_code(countries, country_code); +-} +- + static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { + uint32_t asn = loc_network_get_asn(network); + +@@ -1148,8 +1142,14 @@ static int loc_database_enumerator_filter_network( + return 1; + + // Skip if the country code does not match +- if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries)) +- return 1; ++ if (enumerator->countries) { ++ if (!loc_country_list_empty(enumerator->countries)) { ++ const char* country_code = loc_network_get_country_code(network); ++ ++ if (!loc_country_list_contains_code(enumerator->countries, country_code)) ++ return 1; ++ } ++ } + + // Skip if the ASN does not match + if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) +-- +2.20.1 + +From c1a36c943181da5cd2aef589a972d5027e529eb8 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 17 Nov 2020 16:58:55 +0000 +Subject: [PATCH 61/70] database: Simplify AS matching code + +Signed-off-by: Michael Tremer +--- + src/database.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 1a354f6..be93e00 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node( + return 0; + } + +-static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { +- uint32_t asn = loc_network_get_asn(network); +- +- return loc_as_list_contains_number(asns, asn); +-} +- + static int loc_database_enumerator_filter_network( + struct loc_database_enumerator* enumerator, struct loc_network* network) { + // Skip if the family does not match +@@ -1152,8 +1146,14 @@ static int loc_database_enumerator_filter_network( + } + + // Skip if the ASN does not match +- if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) +- return 1; ++ if (enumerator->asns) { ++ if (!loc_as_list_empty(enumerator->asns)) { ++ uint32_t asn = loc_network_get_asn(network); ++ ++ if (!loc_as_list_contains_number(enumerator->asns, asn)) ++ return 1; ++ } ++ } + + // Skip if flags do not match + if (enumerator->flags && +-- +2.20.1 + +From d5205091f9cc1ff987e483325d48696459df08d8 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 17 Nov 2020 17:50:17 +0000 +Subject: [PATCH 62/70] countries: Make list grow dynamically + +Signed-off-by: Michael Tremer +--- + src/country-list.c | 38 ++++++++++++++++++++++++++------------ + 1 file changed, 26 insertions(+), 12 deletions(-) + +diff --git a/src/country-list.c b/src/country-list.c +index ae0d71a..1ce2d06 100644 +--- a/src/country-list.c ++++ b/src/country-list.c +@@ -25,11 +25,27 @@ struct loc_country_list { + struct loc_ctx* ctx; + int refcount; + +- struct loc_country* list[1024]; ++ struct loc_country** elements; ++ size_t elements_size; ++ + size_t size; +- size_t max_size; + }; + ++static int loc_country_list_grow(struct loc_country_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); ++ ++ struct loc_country** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; ++ ++ list->elements = elements; ++ list->elements_size += size; ++ ++ return 0; ++} ++ + LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, + struct loc_country_list** list) { + struct loc_country_list* l = calloc(1, sizeof(*l)); +@@ -39,9 +55,6 @@ LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, + l->ctx = loc_ref(ctx); + l->refcount = 1; + +- // Do not allow this list to grow larger than this +- l->max_size = 1024; +- + DEBUG(l->ctx, "Country list allocated at %p\n", l); + *list = l; + +@@ -84,7 +97,7 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { + + LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { + for (unsigned int i = 0; i < list->size; i++) +- loc_country_unref(list->list[i]); ++ loc_country_unref(list->elements[i]); + } + + LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { +@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* lis + if (index >= list->size) + return NULL; + +- return loc_country_ref(list->list[index]); ++ return loc_country_ref(list->elements[index]); + } + + LOC_EXPORT int loc_country_list_append( +@@ -101,14 +114,15 @@ LOC_EXPORT int loc_country_list_append( + return 0; + + // Check if we have space left +- if (list->size == list->max_size) { +- ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list); +- return -ENOMEM; ++ if (list->size >= list->elements_size) { ++ int r = loc_country_list_grow(list, 64); ++ if (r) ++ return r; + } + + DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); + +- list->list[list->size++] = loc_country_ref(country); ++ list->elements[list->size++] = loc_country_ref(country); + + return 0; + } +@@ -116,7 +130,7 @@ LOC_EXPORT int loc_country_list_append( + LOC_EXPORT int loc_country_list_contains( + struct loc_country_list* list, struct loc_country* country) { + for (unsigned int i = 0; i < list->size; i++) { +- if (loc_country_cmp(country, list->list[i]) == 0) ++ if (loc_country_cmp(country, list->elements[i]) == 0) + return 1; + } + +-- +2.20.1 + +From 3b44e4211371d2103f89ba8f056b15edb7778fac Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 17 Nov 2020 17:55:51 +0000 +Subject: [PATCH 63/70] networks: Make list grow dynamically + +Signed-off-by: Michael Tremer +--- + src/network-list.c | 60 ++++++++++++++++++++++++++++------------------ + 1 file changed, 37 insertions(+), 23 deletions(-) + +diff --git a/src/network-list.c b/src/network-list.c +index 1f6e80e..4912c02 100644 +--- a/src/network-list.c ++++ b/src/network-list.c +@@ -25,11 +25,27 @@ struct loc_network_list { + struct loc_ctx* ctx; + int refcount; + +- struct loc_network* list[1024]; ++ struct loc_network** elements; ++ size_t elements_size; ++ + size_t size; +- size_t max_size; + }; + ++static int loc_network_list_grow(struct loc_network_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); ++ ++ struct loc_network** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; ++ ++ list->elements = elements; ++ list->elements_size += size; ++ ++ return 0; ++} ++ + LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, + struct loc_network_list** list) { + struct loc_network_list* l = calloc(1, sizeof(*l)); +@@ -39,9 +55,6 @@ LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, + l->ctx = loc_ref(ctx); + l->refcount = 1; + +- // Do not allow this list to grow larger than this +- l->max_size = 1024; +- + DEBUG(l->ctx, "Network list allocated at %p\n", l); + *list = l; + return 0; +@@ -57,7 +70,7 @@ static void loc_network_list_free(struct loc_network_list* list) { + DEBUG(list->ctx, "Releasing network list at %p\n", list); + + for (unsigned int i = 0; i < list->size; i++) +- loc_network_unref(list->list[i]); ++ loc_network_unref(list->elements[i]); + + loc_unref(list->ctx); + free(list); +@@ -84,7 +97,7 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { + + LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { + for (unsigned int i = 0; i < list->size; i++) +- loc_network_unref(list->list[i]); ++ loc_network_unref(list->elements[i]); + + list->size = 0; + } +@@ -94,7 +107,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { + char* s; + + for (unsigned int i = 0; i < list->size; i++) { +- network = list->list[i]; ++ network = list->elements[i]; + + s = loc_network_str(network); + +@@ -108,7 +121,7 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis + if (index >= list->size) + return NULL; + +- return loc_network_ref(list->list[index]); ++ return loc_network_ref(list->elements[index]); + } + + LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { +@@ -117,14 +130,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n + return 0; + + // Check if we have space left +- if (list->size == list->max_size) { +- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); +- return -ENOMEM; ++ if (list->size >= list->elements_size) { ++ int r = loc_network_list_grow(list, 64); ++ if (r) ++ return r; + } + + DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); + +- list->list[list->size++] = loc_network_ref(network); ++ list->elements[list->size++] = loc_network_ref(network); + + return 0; + } +@@ -136,7 +150,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis + return NULL; + } + +- struct loc_network* network = list->list[--list->size]; ++ struct loc_network* network = list->elements[--list->size]; + + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); + +@@ -150,11 +164,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis + return NULL; + } + +- struct loc_network* network = list->list[0]; ++ struct loc_network* network = list->elements[0]; + + // Move all elements to the top of the stack + for (unsigned int i = 0; i < --list->size; i++) { +- list->list[i] = list->list[i+1]; ++ list->elements[i] = list->elements[i+1]; + } + + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); +@@ -164,7 +178,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis + + LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { + for (unsigned int i = 0; i < list->size; i++) { +- if (loc_network_eq(list->list[i], network)) ++ if (loc_network_eq(list->elements[i], network)) + return 1; + } + +@@ -176,11 +190,11 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1 + if (i1 >= list->size || i2 >= list->size) + return; + +- struct loc_network* network1 = list->list[i1]; +- struct loc_network* network2 = list->list[i2]; ++ struct loc_network* network1 = list->elements[i1]; ++ struct loc_network* network2 = list->elements[i2]; + +- list->list[i1] = network2; +- list->list[i2] = network1; ++ list->elements[i1] = network2; ++ list->elements[i2] = network1; + } + + LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { +@@ -200,7 +214,7 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { + swapped = 0; + + for (unsigned int i = 1; i < n; i++) { +- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { ++ if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) { + loc_network_list_swap(list, i-1, i); + swapped = 1; + } +@@ -215,7 +229,7 @@ LOC_EXPORT int loc_network_list_merge( + int r; + + for (unsigned int i = 0; i < other->size; i++) { +- r = loc_network_list_push(self, other->list[i]); ++ r = loc_network_list_push(self, other->elements[i]); + if (r) + return r; + } +-- +2.20.1 + +From 1a415f8c555f4fe9a68eb2a897c4a1fc0d33db25 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 17 Nov 2020 17:57:55 +0000 +Subject: [PATCH 64/70] as: Make lists grow dynamically + +Signed-off-by: Michael Tremer +--- + src/as-list.c | 38 ++++++++++++++++++++++++++------------ + 1 file changed, 26 insertions(+), 12 deletions(-) + +diff --git a/src/as-list.c b/src/as-list.c +index 7c69eb0..17de23e 100644 +--- a/src/as-list.c ++++ b/src/as-list.c +@@ -25,11 +25,27 @@ struct loc_as_list { + struct loc_ctx* ctx; + int refcount; + +- struct loc_as* list[1024]; ++ struct loc_as** elements; ++ size_t elements_size; ++ + size_t size; +- size_t max_size; + }; + ++static int loc_as_list_grow(struct loc_as_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); ++ ++ struct loc_as** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; ++ ++ list->elements = elements; ++ list->elements_size += size; ++ ++ return 0; ++} ++ + LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, + struct loc_as_list** list) { + struct loc_as_list* l = calloc(1, sizeof(*l)); +@@ -39,9 +55,6 @@ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, + l->ctx = loc_ref(ctx); + l->refcount = 1; + +- // Do not allow this list to grow larger than this +- l->max_size = 1024; +- + DEBUG(l->ctx, "AS list allocated at %p\n", l); + *list = l; + +@@ -84,7 +97,7 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { + + LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { + for (unsigned int i = 0; i < list->size; i++) +- loc_as_unref(list->list[i]); ++ loc_as_unref(list->elements[i]); + } + + LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { +@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index + if (index >= list->size) + return NULL; + +- return loc_as_ref(list->list[index]); ++ return loc_as_ref(list->elements[index]); + } + + LOC_EXPORT int loc_as_list_append( +@@ -101,14 +114,15 @@ LOC_EXPORT int loc_as_list_append( + return 0; + + // Check if we have space left +- if (list->size == list->max_size) { +- ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list); +- return -ENOMEM; ++ if (list->size >= list->elements_size) { ++ int r = loc_as_list_grow(list, 64); ++ if (r) ++ return r; + } + + DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); + +- list->list[list->size++] = loc_as_ref(as); ++ list->elements[list->size++] = loc_as_ref(as); + + return 0; + } +@@ -116,7 +130,7 @@ LOC_EXPORT int loc_as_list_append( + LOC_EXPORT int loc_as_list_contains( + struct loc_as_list* list, struct loc_as* as) { + for (unsigned int i = 0; i < list->size; i++) { +- if (loc_as_cmp(as, list->list[i]) == 0) ++ if (loc_as_cmp(as, list->elements[i]) == 0) + return 1; + } + +-- +2.20.1 + +From e6592434ee7836507c1f436ec3b0db3bc81a81b9 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 17 Nov 2020 18:13:49 +0000 +Subject: [PATCH 65/70] export: Change back to use Network objects + +Signed-off-by: Michael Tremer +--- + src/python/export.py | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/src/python/export.py b/src/python/export.py +index f675eb3..67e437f 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -144,9 +144,7 @@ class XTGeoIPOutputWriter(OutputWriter): + def _write_network(self, network): + for address in (network.first_address, network.last_address): + # Convert this into a string of bits +- bytes = socket.inet_pton( +- socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, +- ) ++ bytes = socket.inet_pton(network.family, address) + + self.f.write(bytes) + +-- +2.20.1 + +From 248f5e0419f2349253b8ea96e477c15649fe2173 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 17 Nov 2020 18:14:15 +0000 +Subject: [PATCH 66/70] Actually clear all lists + +Signed-off-by: Michael Tremer +--- + src/as-list.c | 8 ++++++++ + src/country-list.c | 8 ++++++++ + src/network-list.c | 6 ++++++ + 3 files changed, 22 insertions(+) + +diff --git a/src/as-list.c b/src/as-list.c +index 17de23e..76620c7 100644 +--- a/src/as-list.c ++++ b/src/as-list.c +@@ -96,8 +96,16 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { + } + + LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { ++ if (!list->elements) ++ return; ++ + for (unsigned int i = 0; i < list->size; i++) + loc_as_unref(list->elements[i]); ++ ++ free(list->elements); ++ list->elements_size = 0; ++ ++ list->size = 0; + } + + LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { +diff --git a/src/country-list.c b/src/country-list.c +index 1ce2d06..1c49c47 100644 +--- a/src/country-list.c ++++ b/src/country-list.c +@@ -96,8 +96,16 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { + } + + LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { ++ if (!list->elements) ++ return; ++ + for (unsigned int i = 0; i < list->size; i++) + loc_country_unref(list->elements[i]); ++ ++ free(list->elements); ++ list->elements_size = 0; ++ ++ list->size = 0; + } + + LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { +diff --git a/src/network-list.c b/src/network-list.c +index 4912c02..9cb4547 100644 +--- a/src/network-list.c ++++ b/src/network-list.c +@@ -96,9 +96,15 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { + } + + LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { ++ if (!list->elements) ++ return; ++ + for (unsigned int i = 0; i < list->size; i++) + loc_network_unref(list->elements[i]); + ++ free(list->elements); ++ list->elements_size = 0; ++ + list->size = 0; + } + +-- +2.20.1 + +From c98ebf8aae2aa141193db52cd9429b1ded5b09c4 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 17 Nov 2020 18:34:51 +0000 +Subject: [PATCH 67/70] database: Do not clean up python list + +Signed-off-by: Michael Tremer +--- + src/python/database.c | 9 +++++++-- + 1 file changed, 7 insertions(+), 2 deletions(-) + +diff --git a/src/python/database.c b/src/python/database.c +index 38a804c..ed22275 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -325,9 +325,14 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, + loc_country_unref(country); + } + +- loc_database_enumerator_set_countries(enumerator, countries); ++ r = loc_database_enumerator_set_countries(enumerator, countries); ++ if (r) { ++ PyErr_SetFromErrno(PyExc_SystemError); ++ ++ loc_as_list_unref(countries); ++ return NULL; ++ } + +- Py_DECREF(country_codes); + loc_country_list_unref(countries); + } + +-- +2.20.1 + +From 5470d06cb59027f4e04b6d576763dbf7f1093fde Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 17 Nov 2020 19:01:04 +0000 +Subject: [PATCH 68/70] database: Free filter lists in enumerator + +Signed-off-by: Michael Tremer +--- + src/database.c | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/src/database.c b/src/database.c +index be93e00..ca35fe1 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -950,6 +950,12 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera + if (enumerator->string) + free(enumerator->string); + ++ if (enumerator->countries) ++ loc_country_list_unref(enumerator->countries); ++ ++ if (enumerator->asns) ++ loc_as_list_unref(enumerator->asns); ++ + // Free network search + free(enumerator->networks_visited); + +-- +2.20.1 + +From e0e96878d3df51c4a265d51d088005dedf9335e3 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 18 Nov 2020 13:18:52 +0000 +Subject: [PATCH 69/70] database: Add debug output to filtering + +Signed-off-by: Michael Tremer +--- + src/database.c | 29 ++++++++++++++++------------- + 1 file changed, 16 insertions(+), 13 deletions(-) + +diff --git a/src/database.c b/src/database.c +index ca35fe1..83dd752 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1138,33 +1138,36 @@ static int loc_database_enumerator_stack_push_node( + static int loc_database_enumerator_filter_network( + struct loc_database_enumerator* enumerator, struct loc_network* network) { + // Skip if the family does not match +- if (enumerator->family && loc_network_address_family(network) != enumerator->family) ++ if (enumerator->family && loc_network_address_family(network) != enumerator->family) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network); + return 1; ++ } + + // Skip if the country code does not match +- if (enumerator->countries) { +- if (!loc_country_list_empty(enumerator->countries)) { +- const char* country_code = loc_network_get_country_code(network); ++ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) { ++ const char* country_code = loc_network_get_country_code(network); + +- if (!loc_country_list_contains_code(enumerator->countries, country_code)) +- return 1; ++ if (!loc_country_list_contains_code(enumerator->countries, country_code)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network); ++ return 1; + } + } + + // Skip if the ASN does not match +- if (enumerator->asns) { +- if (!loc_as_list_empty(enumerator->asns)) { +- uint32_t asn = loc_network_get_asn(network); ++ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) { ++ uint32_t asn = loc_network_get_asn(network); + +- if (!loc_as_list_contains_number(enumerator->asns, asn)) +- return 1; ++ if (!loc_as_list_contains_number(enumerator->asns, asn)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network); ++ return 1; + } + } + + // Skip if flags do not match +- if (enumerator->flags && +- !loc_network_match_flag(network, enumerator->flags)) ++ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network); + return 1; ++ } + + // Do not filter + return 0; +-- +2.20.1 + +From bce0c9295ff8ff9488f24babe01ce851228d0b1e Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 18 Nov 2020 13:19:04 +0000 +Subject: [PATCH 70/70] export: Remove filtering for flags + +The filter is an AND filter and if we set the flags from +the special country codes, we won't get back much. + +Signed-off-by: Michael Tremer +--- + src/python/export.py | 8 +------- + 1 file changed, 1 insertion(+), 7 deletions(-) + +diff --git a/src/python/export.py b/src/python/export.py +index 67e437f..4219957 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -187,15 +187,9 @@ class Exporter(object): + country_code for country_code in countries if not country_code in FLAGS.values() + ] + +- # Collect flags +- flags = 0 +- for flag in FLAGS: +- if FLAGS[flag] in countries: +- flags |= flag +- + # Get all networks that match the family + networks = self.db.search_networks(family=family, +- country_codes=country_codes, asns=asns, flags=flags, flatten=True) ++ country_codes=country_codes, asns=asns, flatten=True) + + # Walk through all networks + for network in networks: +-- +2.20.1 +