From b952a52b706da0d0cc208dbc6870785d7460c32b Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Thu, 26 Nov 2020 16:15:07 +0000 Subject: [PATCH] libloc: Import latest changes from upstream This is now a unified patch instead of being split into individual commits from upstream. Signed-off-by: Michael Tremer --- src/patches/libloc-0.9.4-upstream.patch | 13050 +++++----------------- 1 file changed, 3041 insertions(+), 10009 deletions(-) diff --git a/src/patches/libloc-0.9.4-upstream.patch b/src/patches/libloc-0.9.4-upstream.patch index 4164205320..79db509037 100644 --- a/src/patches/libloc-0.9.4-upstream.patch +++ b/src/patches/libloc-0.9.4-upstream.patch @@ -1,834 +1,251 @@ -From ee6ea3986dc80183157f67275dc9f28231b5d5b2 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 24 Sep 2020 10:17:58 +0000 -Subject: [PATCH 001/111] Revert "importer: Purge any redundant entries" - -This reverts commit c2cc55d5a6875c3838f060032eaed89dcfb92ef6. - -The query stalls the database and therefore the automatic -scripts are no longer able to generate a new version of the -database. - -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 22 +--------------------- - 1 file changed, 1 insertion(+), 21 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 1467923..e3a07a0 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -374,27 +374,7 @@ class CLI(object): - INSERT INTO autnums(number, name) - SELECT _autnums.number, _organizations.name FROM _autnums - JOIN _organizations ON _autnums.organization = _organizations.handle -- ON CONFLICT (number) DO UPDATE SET name = excluded.name -- """) -- -- self.db.execute(""" -- --- Purge any redundant entries -- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP -- AS -- SELECT network FROM networks candidates -- WHERE EXISTS ( -- SELECT FROM networks -- WHERE -- networks.network << candidates.network -- AND -- networks.country = candidates.country -- ); -- -- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network); -- -- DELETE FROM networks WHERE EXISTS ( -- SELECT FROM _garbage WHERE networks.network = _garbage.network -- ); -+ ON CONFLICT (number) DO UPDATE SET name = excluded.name; - """) - - # Download all extended sources --- -2.20.1 - -From 92f6abf4e272672bb0a71cfe991261b95ebe2fef Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 24 Sep 2020 10:18:58 +0000 -Subject: [PATCH 002/111] Revert "importer: Import raw sources for inetnum's - again" - -This reverts commit 64e95fa903edec8b4e4e59830b395e2e4a411853. - -Signed-off-by: Michael Tremer ---- - src/python/importer.py | 14 ++++---- - src/python/location-importer.in | 63 --------------------------------- - 2 files changed, 7 insertions(+), 70 deletions(-) - -diff --git a/src/python/importer.py b/src/python/importer.py -index f19db4b..de20f37 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( - "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", - - # Asia Pacific Network Information Centre -- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", -+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", - "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", -@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( - # XXX ??? - - # Réseaux IP Européens -- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", -+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", -@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( - - EXTENDED_SOURCES = ( - # African Network Information Centre -- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", -+ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", - - # Asia Pacific Network Information Centre -- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", -+ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", - - # American Registry for Internet Numbers - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", -@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - - # Réseaux IP Européens -- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", -+ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", - ) - - class Downloader(object): -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index e3a07a0..77952f2 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -393,10 +393,6 @@ class CLI(object): - if line.startswith("aut-num:"): - return self._parse_autnum_block(block) - -- # inetnum -- if line.startswith("inet6num:") or line.startswith("inetnum:"): -- return self._parse_inetnum_block(block) -- - # organisation - elif line.startswith("organisation:"): - return self._parse_org_block(block) -@@ -426,65 +422,6 @@ class CLI(object): - autnum.get("asn"), autnum.get("org"), - ) - -- def _parse_inetnum_block(self, block): -- logging.debug("Parsing inetnum block:") -- -- inetnum = {} -- for line in block: -- logging.debug(line) -- -- # Split line -- key, val = split_line(line) -- -- if key == "inetnum": -- start_address, delim, end_address = val.partition("-") -- -- # Strip any excess space -- start_address, end_address = start_address.rstrip(), end_address.strip() -- -- # Convert to IP address -- try: -- start_address = ipaddress.ip_address(start_address) -- end_address = ipaddress.ip_address(end_address) -- except ValueError: -- logging.warning("Could not parse line: %s" % line) -- return -- -- # Set prefix to default -- prefix = 32 -- -- # Count number of addresses in this subnet -- num_addresses = int(end_address) - int(start_address) -- if num_addresses: -- prefix -= math.log(num_addresses, 2) -- -- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) -- -- elif key == "inet6num": -- inetnum[key] = val -- -- elif key == "country": -- if val == "UNITED STATES": -- val = "US" -- -- inetnum[key] = val.upper() -- -- # Skip empty objects -- if not inetnum: -- return -- -- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) -- -- # Bail out in case we have processed a non-public IP network -- if network.is_private: -- logging.warning("Skipping non-globally routable network: %s" % network) -- return -- -- self.db.execute("INSERT INTO networks(network, country) \ -- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", -- "%s" % network, inetnum.get("country"), -- ) -- - def _parse_org_block(self, block): - org = {} - for line in block: --- -2.20.1 - -From f532841e9197ce2f40aad8c086d786b2cb783a54 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Mon, 12 Oct 2020 20:53:31 +0000 -Subject: [PATCH 003/111] Revert "Revert "importer: Import raw sources for - inetnum's again"" - -This reverts commit 92f6abf4e272672bb0a71cfe991261b95ebe2fef. - -Signed-off-by: Michael Tremer ---- - src/python/importer.py | 14 ++++---- - src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++ - 2 files changed, 70 insertions(+), 7 deletions(-) - -diff --git a/src/python/importer.py b/src/python/importer.py -index de20f37..f19db4b 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( - "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", - - # Asia Pacific Network Information Centre -- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", -+ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -+ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", - "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", -@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( - # XXX ??? - - # Réseaux IP Européens -- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", -+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", -@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( - - EXTENDED_SOURCES = ( - # African Network Information Centre -- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", -+ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", - - # Asia Pacific Network Information Centre -- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", -+ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", - - # American Registry for Internet Numbers - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", -@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - - # Réseaux IP Européens -- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", -+ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", - ) +diff --git a/Makefile.am b/Makefile.am +index a0431a6..ebd7e17 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -91,11 +91,14 @@ EXTRA_DIST += \ + pkginclude_HEADERS = \ + src/loc/libloc.h \ + src/loc/as.h \ ++ src/loc/as-list.h \ + src/loc/compat.h \ + src/loc/country.h \ ++ src/loc/country-list.h \ + src/loc/database.h \ + src/loc/format.h \ + src/loc/network.h \ ++ src/loc/network-list.h \ + src/loc/private.h \ + src/loc/stringpool.h \ + src/loc/resolv.h \ +@@ -107,9 +110,12 @@ lib_LTLIBRARIES = \ + src_libloc_la_SOURCES = \ + src/libloc.c \ + src/as.c \ ++ src/as-list.c \ + src/country.c \ ++ src/country-list.c \ + src/database.c \ + src/network.c \ ++ src/network-list.c \ + src/resolv.c \ + src/stringpool.c \ + src/writer.c +@@ -312,6 +318,7 @@ check_PROGRAMS = \ + src/test-database \ + src/test-as \ + src/test-network \ ++ src/test-network-list \ + src/test-country \ + src/test-signature - class Downloader(object): -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 77952f2..e3a07a0 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -393,6 +393,10 @@ class CLI(object): - if line.startswith("aut-num:"): - return self._parse_autnum_block(block) - -+ # inetnum -+ if line.startswith("inet6num:") or line.startswith("inetnum:"): -+ return self._parse_inetnum_block(block) -+ - # organisation - elif line.startswith("organisation:"): - return self._parse_org_block(block) -@@ -422,6 +426,65 @@ class CLI(object): - autnum.get("asn"), autnum.get("org"), - ) +@@ -351,6 +358,15 @@ src_test_network_CFLAGS = \ + src_test_network_LDADD = \ + src/libloc.la -+ def _parse_inetnum_block(self, block): -+ logging.debug("Parsing inetnum block:") ++src_test_network_list_SOURCES = \ ++ src/test-network-list.c + -+ inetnum = {} -+ for line in block: -+ logging.debug(line) ++src_test_network_list_CFLAGS = \ ++ $(TESTS_CFLAGS) + -+ # Split line -+ key, val = split_line(line) ++src_test_network_list_LDADD = \ ++ src/libloc.la + -+ if key == "inetnum": -+ start_address, delim, end_address = val.partition("-") + src_test_stringpool_SOURCES = \ + src/test-stringpool.c + +diff --git a/configure.ac b/configure.ac +index 2364dfd..012d8ca 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1,6 +1,6 @@ + AC_PREREQ(2.60) + AC_INIT([libloc], +- [0.9.4], ++ [0.9.5], + [location@lists.ipfire.org], + [libloc], + [https://location.ipfire.org/]) +diff --git a/src/.gitignore b/src/.gitignore +index caf80b5..3ccbdb8 100644 +--- a/src/.gitignore ++++ b/src/.gitignore +@@ -10,5 +10,6 @@ test-libloc + test-database + test-country + test-network ++test-network-list + test-signature + test-stringpool +diff --git a/src/as-list.c b/src/as-list.c +new file mode 100644 +index 0000000..5acbb8a +--- /dev/null ++++ b/src/as-list.c +@@ -0,0 +1,161 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet + -+ # Strip any excess space -+ start_address, end_address = start_address.rstrip(), end_address.strip() ++ Copyright (C) 2020 IPFire Development Team + -+ # Convert to IP address -+ try: -+ start_address = ipaddress.ip_address(start_address) -+ end_address = ipaddress.ip_address(end_address) -+ except ValueError: -+ logging.warning("Could not parse line: %s" % line) -+ return ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. + -+ # Set prefix to default -+ prefix = 32 ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ + -+ # Count number of addresses in this subnet -+ num_addresses = int(end_address) - int(start_address) -+ if num_addresses: -+ prefix -= math.log(num_addresses, 2) ++#include ++#include + -+ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) ++#include ++#include ++#include + -+ elif key == "inet6num": -+ inetnum[key] = val ++struct loc_as_list { ++ struct loc_ctx* ctx; ++ int refcount; + -+ elif key == "country": -+ if val == "UNITED STATES": -+ val = "US" ++ struct loc_as** elements; ++ size_t elements_size; + -+ inetnum[key] = val.upper() ++ size_t size; ++}; + -+ # Skip empty objects -+ if not inetnum: -+ return ++static int loc_as_list_grow(struct loc_as_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); + -+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) ++ struct loc_as** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; + -+ # Bail out in case we have processed a non-public IP network -+ if network.is_private: -+ logging.warning("Skipping non-globally routable network: %s" % network) -+ return ++ list->elements = elements; ++ list->elements_size += size; + -+ self.db.execute("INSERT INTO networks(network, country) \ -+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", -+ "%s" % network, inetnum.get("country"), -+ ) ++ return 0; ++} + - def _parse_org_block(self, block): - org = {} - for line in block: --- -2.20.1 - -From a36bc686865fc87ea386fd90b389338bdcb80cbc Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Mon, 12 Oct 2020 20:53:32 +0000 -Subject: [PATCH 004/111] location-importer.in: only import relevant data from - AFRINIC, APNIC and RIPE -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -In contrast to ARIN and LACNIC, we are able to process more detailled -feeds from those RIRs, avoiding storage of obviously unnecessary data. - -Thanks to various SQL optimisations, doing so now takes less time than -the first version of this did. - -Signed-off-by: Michael Tremer -Signed-off-by: Peter Müller -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++- - 1 file changed, 87 insertions(+), 2 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index e3a07a0..093f325 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -165,6 +165,7 @@ class CLI(object): - -- networks - CREATE TABLE IF NOT EXISTS networks(network inet, country text); - CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); -+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); - CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); - - -- overrides -@@ -363,6 +364,16 @@ class CLI(object): - CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) - ON COMMIT DROP; - CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); ++LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, ++ struct loc_as_list** list) { ++ struct loc_as_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; + -+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) -+ ON COMMIT DROP; -+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); -+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); -+ """) ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; + -+ # Remove all previously imported content -+ self.db.execute(""" -+ TRUNCATE TABLE networks; - """) - - for source in location.importer.WHOIS_SOURCES: -@@ -370,6 +381,67 @@ class CLI(object): - for block in f: - self._parse_block(block) - -+ # Process all parsed networks from every RIR we happen to have access to, -+ # insert the largest network chunks into the networks table immediately... -+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") ++ DEBUG(l->ctx, "AS list allocated at %p\n", l); ++ *list = l; + -+ for family in (row.family for row in families): -+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) ++ return 0; ++} + -+ self.db.execute("INSERT INTO networks(network, country) \ -+ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) ++LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) { ++ list->refcount++; + -+ # ... determine any other prefixes for this network family, ... -+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ -+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) ++ return list; ++} + -+ # ... and insert networks with this prefix in case they provide additional -+ # information (i. e. subnet of a larger chunk with a different country) -+ for prefix in (row.prefix for row in prefixes): -+ self.db.execute(""" -+ WITH candidates AS ( -+ SELECT -+ _rirdata.network, -+ _rirdata.country -+ FROM -+ _rirdata -+ WHERE -+ family(_rirdata.network) = %s -+ AND -+ masklen(_rirdata.network) = %s -+ ), -+ filtered AS ( -+ SELECT -+ DISTINCT ON (c.network) -+ c.network, -+ c.country, -+ masklen(networks.network), -+ networks.country AS parent_country -+ FROM -+ candidates c -+ LEFT JOIN -+ networks -+ ON -+ c.network << networks.network -+ ORDER BY -+ c.network, -+ masklen(networks.network) DESC NULLS LAST -+ ) -+ INSERT INTO -+ networks(network, country) -+ SELECT -+ network, -+ country -+ FROM -+ filtered -+ WHERE -+ parent_country IS NULL -+ OR -+ country <> parent_country -+ ON CONFLICT DO NOTHING""", -+ family, prefix, -+ ) ++static void loc_as_list_free(struct loc_as_list* list) { ++ DEBUG(list->ctx, "Releasing AS list at %p\n", list); + - self.db.execute(""" - INSERT INTO autnums(number, name) - SELECT _autnums.number, _organizations.name FROM _autnums -@@ -470,17 +542,30 @@ class CLI(object): - inetnum[key] = val.upper() - - # Skip empty objects -- if not inetnum: -+ if not inetnum or not "country" in inetnum: - return - - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) - -+ # Bail out in case we have processed a network covering the entire IP range, which -+ # is necessary to work around faulty (?) IPv6 network processing -+ if network.prefixlen == 0: -+ logging.warning("Skipping network covering the entire IP adress range: %s" % network) -+ return ++ loc_as_list_clear(list); + -+ # Bail out in case we have processed a network whose prefix length indicates it is -+ # not globally routable (we have decided not to process them at the moment, as they -+ # significantly enlarge our database without providing very helpful additional information) -+ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): -+ logging.info("Skipping network too small to be publicly announced: %s" % network) -+ return ++ loc_unref(list->ctx); ++ free(list); ++} + - # Bail out in case we have processed a non-public IP network - if network.is_private: - logging.warning("Skipping non-globally routable network: %s" % network) - return - -- self.db.execute("INSERT INTO networks(network, country) \ -+ self.db.execute("INSERT INTO _rirdata(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", - "%s" % network, inetnum.get("country"), - ) --- -2.20.1 - -From 2373de384f10f5573bbd7570f5522545df70c0e3 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 16 Oct 2020 12:24:58 +0000 -Subject: [PATCH 005/111] location-importer: Include all overridden networks - -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 093f325..d249a35 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -240,6 +240,8 @@ class CLI(object): - SELECT network FROM announcements - UNION - SELECT network FROM networks -+ UNION -+ SELECT network FROM network_overrides - ORDER BY network - ) - --- -2.20.1 - -From 13f67f285856e8eabfeff2daf1be3aeaa36a82cc Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 16 Oct 2020 12:26:38 +0000 -Subject: [PATCH 006/111] Revert "location-importer.in: only import relevant - data from AFRINIC, APNIC and RIPE" - -This reverts commit a36bc686865fc87ea386fd90b389338bdcb80cbc. - -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 89 +-------------------------------- - 1 file changed, 2 insertions(+), 87 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index d249a35..b220eaf 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -165,7 +165,6 @@ class CLI(object): - -- networks - CREATE TABLE IF NOT EXISTS networks(network inet, country text); - CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); -- CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); - CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); - - -- overrides -@@ -366,16 +365,6 @@ class CLI(object): - CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) - ON COMMIT DROP; - CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); -- -- CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) -- ON COMMIT DROP; -- CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); -- CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); -- """) -- -- # Remove all previously imported content -- self.db.execute(""" -- TRUNCATE TABLE networks; - """) - - for source in location.importer.WHOIS_SOURCES: -@@ -383,67 +372,6 @@ class CLI(object): - for block in f: - self._parse_block(block) - -- # Process all parsed networks from every RIR we happen to have access to, -- # insert the largest network chunks into the networks table immediately... -- families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") -- -- for family in (row.family for row in families): -- smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) -- -- self.db.execute("INSERT INTO networks(network, country) \ -- SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) -- -- # ... determine any other prefixes for this network family, ... -- prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ -- WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) -- -- # ... and insert networks with this prefix in case they provide additional -- # information (i. e. subnet of a larger chunk with a different country) -- for prefix in (row.prefix for row in prefixes): -- self.db.execute(""" -- WITH candidates AS ( -- SELECT -- _rirdata.network, -- _rirdata.country -- FROM -- _rirdata -- WHERE -- family(_rirdata.network) = %s -- AND -- masklen(_rirdata.network) = %s -- ), -- filtered AS ( -- SELECT -- DISTINCT ON (c.network) -- c.network, -- c.country, -- masklen(networks.network), -- networks.country AS parent_country -- FROM -- candidates c -- LEFT JOIN -- networks -- ON -- c.network << networks.network -- ORDER BY -- c.network, -- masklen(networks.network) DESC NULLS LAST -- ) -- INSERT INTO -- networks(network, country) -- SELECT -- network, -- country -- FROM -- filtered -- WHERE -- parent_country IS NULL -- OR -- country <> parent_country -- ON CONFLICT DO NOTHING""", -- family, prefix, -- ) -- - self.db.execute(""" - INSERT INTO autnums(number, name) - SELECT _autnums.number, _organizations.name FROM _autnums -@@ -544,30 +472,17 @@ class CLI(object): - inetnum[key] = val.upper() - - # Skip empty objects -- if not inetnum or not "country" in inetnum: -+ if not inetnum: - return - - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) - -- # Bail out in case we have processed a network covering the entire IP range, which -- # is necessary to work around faulty (?) IPv6 network processing -- if network.prefixlen == 0: -- logging.warning("Skipping network covering the entire IP adress range: %s" % network) -- return -- -- # Bail out in case we have processed a network whose prefix length indicates it is -- # not globally routable (we have decided not to process them at the moment, as they -- # significantly enlarge our database without providing very helpful additional information) -- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): -- logging.info("Skipping network too small to be publicly announced: %s" % network) -- return -- - # Bail out in case we have processed a non-public IP network - if network.is_private: - logging.warning("Skipping non-globally routable network: %s" % network) - return - -- self.db.execute("INSERT INTO _rirdata(network, country) \ -+ self.db.execute("INSERT INTO networks(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", - "%s" % network, inetnum.get("country"), - ) --- -2.20.1 - -From 44341478233115b26bb27fdb24da5b0a1eedb173 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 16 Oct 2020 12:26:43 +0000 -Subject: [PATCH 007/111] Revert "Revert "Revert "importer: Import raw sources - for inetnum's again""" - -This reverts commit f532841e9197ce2f40aad8c086d786b2cb783a54. - -Signed-off-by: Michael Tremer ---- - src/python/importer.py | 14 ++++---- - src/python/location-importer.in | 63 --------------------------------- - 2 files changed, 7 insertions(+), 70 deletions(-) - -diff --git a/src/python/importer.py b/src/python/importer.py -index f19db4b..de20f37 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( - "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", - - # Asia Pacific Network Information Centre -- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", -+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", - "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", -@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( - # XXX ??? - - # Réseaux IP Européens -- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", -+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", -@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( - - EXTENDED_SOURCES = ( - # African Network Information Centre -- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", -+ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", - - # Asia Pacific Network Information Centre -- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", -+ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", - - # American Registry for Internet Numbers - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", -@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - - # Réseaux IP Européens -- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", -+ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", - ) - - class Downloader(object): -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index b220eaf..e87d378 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -395,10 +395,6 @@ class CLI(object): - if line.startswith("aut-num:"): - return self._parse_autnum_block(block) - -- # inetnum -- if line.startswith("inet6num:") or line.startswith("inetnum:"): -- return self._parse_inetnum_block(block) -- - # organisation - elif line.startswith("organisation:"): - return self._parse_org_block(block) -@@ -428,65 +424,6 @@ class CLI(object): - autnum.get("asn"), autnum.get("org"), - ) - -- def _parse_inetnum_block(self, block): -- logging.debug("Parsing inetnum block:") -- -- inetnum = {} -- for line in block: -- logging.debug(line) -- -- # Split line -- key, val = split_line(line) -- -- if key == "inetnum": -- start_address, delim, end_address = val.partition("-") -- -- # Strip any excess space -- start_address, end_address = start_address.rstrip(), end_address.strip() -- -- # Convert to IP address -- try: -- start_address = ipaddress.ip_address(start_address) -- end_address = ipaddress.ip_address(end_address) -- except ValueError: -- logging.warning("Could not parse line: %s" % line) -- return -- -- # Set prefix to default -- prefix = 32 -- -- # Count number of addresses in this subnet -- num_addresses = int(end_address) - int(start_address) -- if num_addresses: -- prefix -= math.log(num_addresses, 2) -- -- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) -- -- elif key == "inet6num": -- inetnum[key] = val -- -- elif key == "country": -- if val == "UNITED STATES": -- val = "US" -- -- inetnum[key] = val.upper() -- -- # Skip empty objects -- if not inetnum: -- return -- -- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) -- -- # Bail out in case we have processed a non-public IP network -- if network.is_private: -- logging.warning("Skipping non-globally routable network: %s" % network) -- return -- -- self.db.execute("INSERT INTO networks(network, country) \ -- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", -- "%s" % network, inetnum.get("country"), -- ) -- - def _parse_org_block(self, block): - org = {} - for line in block: --- -2.20.1 - -From a7d3a7a0565a0e09d3442e5829a0f30f016993b9 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 20 Oct 2020 20:44:43 +0000 -Subject: [PATCH 008/111] as: Fix dereferencing NULL pointer when setting AS - name - -Reported-by: Gisle Vanem -Signed-off-by: Michael Tremer ---- - src/as.c | 8 +++++++- - 1 file changed, 7 insertions(+), 1 deletion(-) - ++LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) { ++ if (!list) ++ return NULL; ++ ++ if (--list->refcount > 0) ++ return list; ++ ++ loc_as_list_free(list); ++ return NULL; ++} ++ ++LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) { ++ return list->size; ++} ++ ++LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { ++ return list->size == 0; ++} ++ ++LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { ++ if (!list->elements) ++ return; ++ ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_as_unref(list->elements[i]); ++ ++ free(list->elements); ++ list->elements = NULL; ++ list->elements_size = 0; ++ ++ list->size = 0; ++} ++ ++LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; ++ ++ return loc_as_ref(list->elements[index]); ++} ++ ++LOC_EXPORT int loc_as_list_append( ++ struct loc_as_list* list, struct loc_as* as) { ++ if (loc_as_list_contains(list, as)) ++ return 0; ++ ++ // Check if we have space left ++ if (list->size >= list->elements_size) { ++ int r = loc_as_list_grow(list, 64); ++ if (r) ++ return r; ++ } ++ ++ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); ++ ++ list->elements[list->size++] = loc_as_ref(as); ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_as_list_contains( ++ struct loc_as_list* list, struct loc_as* as) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_as_cmp(as, list->elements[i]) == 0) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_as_list_contains_number( ++ struct loc_as_list* list, uint32_t number) { ++ struct loc_as* as; ++ ++ int r = loc_as_new(list->ctx, &as, number); ++ if (r) ++ return -1; ++ ++ r = loc_as_list_contains(list, as); ++ loc_as_unref(as); ++ ++ return r; ++} diff --git a/src/as.c b/src/as.c -index e1fbb01..8421ac8 100644 +index e1fbb01..757bf3d 100644 --- a/src/as.c +++ b/src/as.c @@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) { @@ -846,24 +263,7 @@ index e1fbb01..8421ac8 100644 return 0; } --- -2.20.1 - -From ddb326ad38a7c7202315dd2c6f938313db04ee22 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 21 Oct 2020 09:18:08 +0000 -Subject: [PATCH 009/111] as: Do not attempt to match name when it wasn't set - -Signed-off-by: Michael Tremer ---- - src/as.c | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/src/as.c b/src/as.c -index 8421ac8..757bf3d 100644 ---- a/src/as.c -+++ b/src/as.c -@@ -145,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) { +@@ -139,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) { if (!string) return 1; @@ -874,4414 +274,857 @@ index 8421ac8..757bf3d 100644 // Search if string is in name if (strcasestr(as->name, string) != NULL) return 1; --- -2.20.1 - -From d226ad2d97cbcd42ce807d9308569b1b9c5d4e2f Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 21 Oct 2020 09:28:39 +0000 -Subject: [PATCH 010/111] writer: Free array with pointer to ASes, too - -Signed-off-by: Michael Tremer ---- - src/writer.c | 7 +++++-- - 1 file changed, 5 insertions(+), 2 deletions(-) - -diff --git a/src/writer.c b/src/writer.c -index 5939cff..160650f 100644 ---- a/src/writer.c -+++ b/src/writer.c -@@ -147,8 +147,11 @@ static void loc_writer_free(struct loc_writer* writer) { - EVP_PKEY_free(writer->private_key2); - - // Unref all AS -- for (unsigned int i = 0; i < writer->as_count; i++) { -- loc_as_unref(writer->as[i]); -+ if (writer->as) { -+ for (unsigned int i = 0; i < writer->as_count; i++) { -+ loc_as_unref(writer->as[i]); -+ } -+ free(writer->as); - } - - // Release network tree --- -2.20.1 - -From d89a7d62772048ae1bd18d03f69df46b7e1a3d3c Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 21 Oct 2020 09:31:29 +0000 -Subject: [PATCH 011/111] writer: Free countries when the writer is being - destroyed - -Signed-off-by: Michael Tremer ---- - src/writer.c | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/src/writer.c b/src/writer.c -index 160650f..2f09b56 100644 ---- a/src/writer.c -+++ b/src/writer.c -@@ -154,6 +154,14 @@ static void loc_writer_free(struct loc_writer* writer) { - free(writer->as); - } - -+ // Unref all countries -+ if (writer->countries) { -+ for (unsigned int i = 0; i < writer->countries_count; i++) { -+ loc_country_unref(writer->countries[i]); -+ } -+ free(writer->countries); -+ } -+ - // Release network tree - if (writer->networks) - loc_network_tree_unref(writer->networks); --- -2.20.1 - -From 0f1aedbc68e3945770c93e0ebd83eed0f555d6f0 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 21 Oct 2020 13:19:44 +0000 -Subject: [PATCH 012/111] tests: Try adding an invalid network - -Signed-off-by: Michael Tremer ---- - src/test-network.c | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/src/test-network.c b/src/test-network.c -index d38f13d..e908b57 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -160,6 +160,14 @@ int main(int argc, char** argv) { - // Set ASN - loc_network_set_asn(network4, 1024); - -+ // Try adding an invalid network -+ struct loc_network* network; -+ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32"); -+ if (err != -EINVAL) { -+ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); -+ exit(EXIT_FAILURE); -+ } -+ - FILE* f = tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); --- -2.20.1 - -From 13ad6e695f9ffc7847b3afe3e9cbcea8fb3a443f Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 21 Oct 2020 13:36:35 +0000 -Subject: [PATCH 013/111] networks: Improve parsing IP addresses - -loc_network_new_from_string() seem to have had some unexpected -behaviour for invalid inputs. - -The function has been tidied up slightly and returns as soon as -some invalid input was detected. - -Signed-off-by: Michael Tremer ---- - src/network.c | 52 +++++++++++++++++++++++++++++++--------------- - src/test-network.c | 7 +++++++ - 2 files changed, 42 insertions(+), 17 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 366caa2..c112a41 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -160,9 +160,10 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network - LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network, - const char* address_string) { - struct in6_addr first_address; -- unsigned int prefix = 0; - char* prefix_string; -- int r = 1; -+ int r = -EINVAL; -+ -+ DEBUG(ctx, "Attempting to parse network %s\n", address_string); - - // Make a copy of the string to work on it - char* buffer = strdup(address_string); -@@ -171,29 +172,46 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo - // Split address and prefix - address_string = strsep(&prefix_string, "/"); - -- // Did we find a prefix? -- if (prefix_string) { -- // Convert prefix to integer -- prefix = strtol(prefix_string, NULL, 10); -+ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string); - -- if (prefix) { -- // Parse the address -- r = loc_parse_address(ctx, address_string, &first_address); -+ // We need to have a prefix -+ if (!prefix_string) { -+ DEBUG(ctx, "No prefix set\n"); -+ goto FAIL; -+ } - -- // Map the prefix to IPv6 if needed -- if (IN6_IS_ADDR_V4MAPPED(&first_address)) -- prefix += 96; -- } -+ // Convert prefix to integer -+ unsigned int prefix = strtol(prefix_string, NULL, 10); -+ -+ // End if the prefix was invalid -+ if (!prefix) { -+ DEBUG(ctx, "The prefix is zero or not a number\n"); -+ goto FAIL; -+ } -+ -+ // Parse the address -+ r = loc_parse_address(ctx, address_string, &first_address); -+ if (r) { -+ DEBUG(ctx, "The address could not be parsed\n"); -+ goto FAIL; - } - -+ // Map the prefix to IPv6 if needed -+ if (IN6_IS_ADDR_V4MAPPED(&first_address)) -+ prefix += 96; -+ -+FAIL: - // Free temporary buffer - free(buffer); - -- if (r == 0) { -- r = loc_network_new(ctx, network, &first_address, prefix); -- } -+ // Exit if the parsing was unsuccessful -+ if (r) -+ return r; -+ -+ DEBUG(ctx, "GOT HERE\n"); - -- return r; -+ // Create a new network -+ return loc_network_new(ctx, network, &first_address, prefix); - } - - LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) { -diff --git a/src/test-network.c b/src/test-network.c -index e908b57..85eca00 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -168,6 +168,13 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ // Try adding a single address -+ err = loc_writer_add_network(writer, &network, "2001:db8::"); -+ if (err != -EINVAL) { -+ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); -+ exit(EXIT_FAILURE); -+ } -+ - FILE* f = tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); --- -2.20.1 - -From 6a467e9345bb5a3d37911c9aaac30019eaa4492b Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 21 Oct 2020 13:43:21 +0000 -Subject: [PATCH 014/111] networks: Test if we can add localhost (IPv6) - -Signed-off-by: Michael Tremer ---- - src/test-network.c | 7 +++++++ - 1 file changed, 7 insertions(+) - -diff --git a/src/test-network.c b/src/test-network.c -index 85eca00..8c7e898 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -175,6 +175,13 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ // Try adding localhost -+ err = loc_writer_add_network(writer, &network, "::1/128"); -+ if (err != -EINVAL) { -+ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ - FILE* f = tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); --- -2.20.1 - -From fc1190aa11e3ff3d2dbf5f4d408c298e7916f46f Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 21 Oct 2020 13:44:50 +0000 -Subject: [PATCH 015/111] networks: Remove accidentially committed debug line - -Signed-off-by: Michael Tremer ---- - src/network.c | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/src/network.c b/src/network.c -index c112a41..be88d75 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -208,8 +208,6 @@ FAIL: - if (r) - return r; - -- DEBUG(ctx, "GOT HERE\n"); -- - // Create a new network - return loc_network_new(ctx, network, &first_address, prefix); - } --- -2.20.1 - -From a1707d8983898b6878cdd5c68744bcc444e278ed Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 21 Oct 2020 13:53:36 +0000 -Subject: [PATCH 016/111] importer: Add search index to announcements table - -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index e87d378..d0fe5a6 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -152,6 +152,7 @@ class CLI(object): - last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); - CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network); - CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network)); -+ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops); - - -- autnums - CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); --- -2.20.1 - -From 991baf530d47adb2ed7a15b65dc4565d07fa6d07 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 21 Oct 2020 13:54:45 +0000 -Subject: [PATCH 017/111] importer: Add search index to network_overrides table - -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index d0fe5a6..fe21d73 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -189,6 +189,8 @@ class CLI(object): - ); - CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network - ON network_overrides(network); -+ CREATE INDEX IF NOT EXISTS network_overrides_search -+ ON network_overrides USING GIST(network inet_ops); - """) - - return db --- -2.20.1 - -From bbea93a74651df10e2ffdbd09eb434dc6a0471bc Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 21 Oct 2020 16:01:57 +0000 -Subject: [PATCH 018/111] importer: Restructure SQL query to be executed in - parallel - -There are no functional changes, this just runs quicker now. - -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 67 ++++++++++++++++++--------------- - 1 file changed, 37 insertions(+), 30 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index fe21d73..3c1e5e2 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -237,34 +237,24 @@ class CLI(object): - - # Select all known networks - rows = self.db.query(""" -- -- Get a (sorted) list of all known networks -- WITH known_networks AS ( -- SELECT network FROM announcements -- UNION -- SELECT network FROM networks -- UNION -- SELECT network FROM network_overrides -- ORDER BY network -- ) -- - -- Return a list of those networks enriched with all - -- other information that we store in the database - SELECT -- DISTINCT ON (known_networks.network) -- known_networks.network AS network, -- announcements.autnum AS autnum, -+ DISTINCT ON (network) -+ network, -+ autnum, - - -- Country - COALESCE( - ( - SELECT country FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT country FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - networks.country - ) AS country, -@@ -273,50 +263,67 @@ class CLI(object): - COALESCE( - ( - SELECT is_anonymous_proxy FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_anonymous_proxy FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - FALSE - ) AS is_anonymous_proxy, - COALESCE( - ( - SELECT is_satellite_provider FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_satellite_provider FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - FALSE - ) AS is_satellite_provider, - COALESCE( - ( - SELECT is_anycast FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_anycast FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - FALSE -- ) AS is_anycast, -- -- -- Must be part of returned values for ORDER BY clause -- masklen(announcements.network) AS sort_a, -- masklen(networks.network) AS sort_b -- FROM known_networks -- LEFT JOIN announcements ON known_networks.network <<= announcements.network -- LEFT JOIN networks ON known_networks.network <<= networks.network -- ORDER BY known_networks.network, sort_a DESC, sort_b DESC -+ ) AS is_anycast -+ FROM ( -+ SELECT -+ known_networks.network AS network, -+ announcements.autnum AS autnum, -+ networks.country AS country, -+ -+ -- Must be part of returned values for ORDER BY clause -+ masklen(announcements.network) AS sort_a, -+ masklen(networks.network) AS sort_b -+ FROM ( -+ SELECT network FROM announcements -+ UNION ALL -+ SELECT network FROM networks -+ UNION ALL -+ SELECT network FROM network_overrides -+ ) known_networks -+ LEFT JOIN -+ announcements ON known_networks.network <<= announcements.network -+ LEFT JOIN -+ networks ON known_networks.network <<= networks.network -+ ORDER BY -+ known_networks.network, -+ sort_a DESC, -+ sort_b DESC -+ ) networks - """) - - for row in rows: --- -2.20.1 - -From 26ab419b68d166f932db1f97c38cb9d793d04187 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 22 Oct 2020 12:24:34 +0000 -Subject: [PATCH 019/111] network: Allow adding single IP addresses and - automatically add the prefix - -Signed-off-by: Michael Tremer ---- - src/network.c | 33 +++++++++++++++------------------ - src/test-network.c | 4 ++-- - 2 files changed, 17 insertions(+), 20 deletions(-) - -diff --git a/src/network.c b/src/network.c -index be88d75..d7b1645 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -161,6 +161,7 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo - const char* address_string) { - struct in6_addr first_address; - char* prefix_string; -+ unsigned int prefix = 128; - int r = -EINVAL; - - DEBUG(ctx, "Attempting to parse network %s\n", address_string); -@@ -174,21 +175,6 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo - - DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string); - -- // We need to have a prefix -- if (!prefix_string) { -- DEBUG(ctx, "No prefix set\n"); -- goto FAIL; -- } -- -- // Convert prefix to integer -- unsigned int prefix = strtol(prefix_string, NULL, 10); -- -- // End if the prefix was invalid -- if (!prefix) { -- DEBUG(ctx, "The prefix is zero or not a number\n"); -- goto FAIL; -- } -- - // Parse the address - r = loc_parse_address(ctx, address_string, &first_address); - if (r) { -@@ -196,9 +182,20 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo - goto FAIL; - } - -- // Map the prefix to IPv6 if needed -- if (IN6_IS_ADDR_V4MAPPED(&first_address)) -- prefix += 96; -+ // If a prefix was given, we will try to parse it -+ if (prefix_string) { -+ // Convert prefix to integer -+ prefix = strtol(prefix_string, NULL, 10); -+ -+ if (!prefix) { -+ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string); -+ goto FAIL; -+ } -+ -+ // Map the prefix to IPv6 if needed -+ if (IN6_IS_ADDR_V4MAPPED(&first_address)) -+ prefix += 96; -+ } - - FAIL: - // Free temporary buffer -diff --git a/src/test-network.c b/src/test-network.c -index 8c7e898..b6776b4 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -170,8 +170,8 @@ int main(int argc, char** argv) { - - // Try adding a single address - err = loc_writer_add_network(writer, &network, "2001:db8::"); -- if (err != -EINVAL) { -- fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); -+ if (err) { -+ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err); - exit(EXIT_FAILURE); - } - --- -2.20.1 - -From aadac4c569e921be1d28dd3b2377ac7f3732213e Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Wed, 21 Oct 2020 14:47:36 +0000 -Subject: [PATCH 020/111] Revert "Revert "Revert "Revert "importer: Import raw - sources for inetnum's again"""" -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This reverts commit 44341478233115b26bb27fdb24da5b0a1eedb173. - -Signed-off-by: Peter Müller -Signed-off-by: Michael Tremer ---- - src/python/importer.py | 14 ++++---- - src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++ - 2 files changed, 70 insertions(+), 7 deletions(-) - -diff --git a/src/python/importer.py b/src/python/importer.py -index de20f37..f19db4b 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( - "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", - - # Asia Pacific Network Information Centre -- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", -+ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -+ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", - "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", -@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( - # XXX ??? - - # Réseaux IP Européens -- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", -+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", -@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( - - EXTENDED_SOURCES = ( - # African Network Information Centre -- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", -+ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", - - # Asia Pacific Network Information Centre -- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", -+ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", - - # American Registry for Internet Numbers - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", -@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - - # Réseaux IP Européens -- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", -+ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", - ) - - class Downloader(object): -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 3c1e5e2..e8a4fc5 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -405,6 +405,10 @@ class CLI(object): - if line.startswith("aut-num:"): - return self._parse_autnum_block(block) - -+ # inetnum -+ if line.startswith("inet6num:") or line.startswith("inetnum:"): -+ return self._parse_inetnum_block(block) -+ - # organisation - elif line.startswith("organisation:"): - return self._parse_org_block(block) -@@ -434,6 +438,65 @@ class CLI(object): - autnum.get("asn"), autnum.get("org"), - ) - -+ def _parse_inetnum_block(self, block): -+ logging.debug("Parsing inetnum block:") -+ -+ inetnum = {} -+ for line in block: -+ logging.debug(line) -+ -+ # Split line -+ key, val = split_line(line) -+ -+ if key == "inetnum": -+ start_address, delim, end_address = val.partition("-") -+ -+ # Strip any excess space -+ start_address, end_address = start_address.rstrip(), end_address.strip() -+ -+ # Convert to IP address -+ try: -+ start_address = ipaddress.ip_address(start_address) -+ end_address = ipaddress.ip_address(end_address) -+ except ValueError: -+ logging.warning("Could not parse line: %s" % line) -+ return -+ -+ # Set prefix to default -+ prefix = 32 -+ -+ # Count number of addresses in this subnet -+ num_addresses = int(end_address) - int(start_address) -+ if num_addresses: -+ prefix -= math.log(num_addresses, 2) -+ -+ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) -+ -+ elif key == "inet6num": -+ inetnum[key] = val -+ -+ elif key == "country": -+ if val == "UNITED STATES": -+ val = "US" -+ -+ inetnum[key] = val.upper() -+ -+ # Skip empty objects -+ if not inetnum: -+ return -+ -+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) -+ -+ # Bail out in case we have processed a non-public IP network -+ if network.is_private: -+ logging.warning("Skipping non-globally routable network: %s" % network) -+ return -+ -+ self.db.execute("INSERT INTO networks(network, country) \ -+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", -+ "%s" % network, inetnum.get("country"), -+ ) -+ - def _parse_org_block(self, block): - org = {} - for line in block: --- -2.20.1 - -From 002deb6b42ac0b3624c07e3352cebd72dc0685a2 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Wed, 21 Oct 2020 14:47:37 +0000 -Subject: [PATCH 021/111] Revert "Revert "location-importer.in: only import - relevant data from AFRINIC, APNIC and RIPE"" -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This reverts commit 13f67f285856e8eabfeff2daf1be3aeaa36a82cc. - -Signed-off-by: Peter Müller -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++- - 1 file changed, 87 insertions(+), 2 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index e8a4fc5..5656c41 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -166,6 +166,7 @@ class CLI(object): - -- networks - CREATE TABLE IF NOT EXISTS networks(network inet, country text); - CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); -+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); - CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); - - -- overrides -@@ -375,6 +376,16 @@ class CLI(object): - CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) - ON COMMIT DROP; - CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); -+ -+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) -+ ON COMMIT DROP; -+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); -+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); -+ """) -+ -+ # Remove all previously imported content -+ self.db.execute(""" -+ TRUNCATE TABLE networks; - """) - - for source in location.importer.WHOIS_SOURCES: -@@ -382,6 +393,67 @@ class CLI(object): - for block in f: - self._parse_block(block) - -+ # Process all parsed networks from every RIR we happen to have access to, -+ # insert the largest network chunks into the networks table immediately... -+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") -+ -+ for family in (row.family for row in families): -+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) -+ -+ self.db.execute("INSERT INTO networks(network, country) \ -+ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) -+ -+ # ... determine any other prefixes for this network family, ... -+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ -+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) -+ -+ # ... and insert networks with this prefix in case they provide additional -+ # information (i. e. subnet of a larger chunk with a different country) -+ for prefix in (row.prefix for row in prefixes): -+ self.db.execute(""" -+ WITH candidates AS ( -+ SELECT -+ _rirdata.network, -+ _rirdata.country -+ FROM -+ _rirdata -+ WHERE -+ family(_rirdata.network) = %s -+ AND -+ masklen(_rirdata.network) = %s -+ ), -+ filtered AS ( -+ SELECT -+ DISTINCT ON (c.network) -+ c.network, -+ c.country, -+ masklen(networks.network), -+ networks.country AS parent_country -+ FROM -+ candidates c -+ LEFT JOIN -+ networks -+ ON -+ c.network << networks.network -+ ORDER BY -+ c.network, -+ masklen(networks.network) DESC NULLS LAST -+ ) -+ INSERT INTO -+ networks(network, country) -+ SELECT -+ network, -+ country -+ FROM -+ filtered -+ WHERE -+ parent_country IS NULL -+ OR -+ country <> parent_country -+ ON CONFLICT DO NOTHING""", -+ family, prefix, -+ ) -+ - self.db.execute(""" - INSERT INTO autnums(number, name) - SELECT _autnums.number, _organizations.name FROM _autnums -@@ -482,17 +554,30 @@ class CLI(object): - inetnum[key] = val.upper() - - # Skip empty objects -- if not inetnum: -+ if not inetnum or not "country" in inetnum: - return - - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) - -+ # Bail out in case we have processed a network covering the entire IP range, which -+ # is necessary to work around faulty (?) IPv6 network processing -+ if network.prefixlen == 0: -+ logging.warning("Skipping network covering the entire IP adress range: %s" % network) -+ return -+ -+ # Bail out in case we have processed a network whose prefix length indicates it is -+ # not globally routable (we have decided not to process them at the moment, as they -+ # significantly enlarge our database without providing very helpful additional information) -+ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): -+ logging.info("Skipping network too small to be publicly announced: %s" % network) -+ return -+ - # Bail out in case we have processed a non-public IP network - if network.is_private: - logging.warning("Skipping non-globally routable network: %s" % network) - return - -- self.db.execute("INSERT INTO networks(network, country) \ -+ self.db.execute("INSERT INTO _rirdata(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", - "%s" % network, inetnum.get("country"), - ) --- -2.20.1 - -From 28c73fa3f4257e0a41e52af8a9643da414a6cb6f Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Wed, 21 Oct 2020 14:47:38 +0000 -Subject: [PATCH 022/111] export.py: fix exporting IP networks for crappy - xt_geoip module -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -In contrast to the location database itself, the xt_geoip module -consumes a list of IP networks for each country, and returns after the -first match. - -We therefore need to... - -(a) sort IP networks by their size, allow as precise matches as possible -(b) export _any_ IP networks - including inverted subnets - to prevent - undefined overlaps -(c) do the entire thing as fast as possible, consuming as less disk - space as possible, which is why we can't just iterate over all /24 - chunks - -Partially fixes: #12499 - -Signed-off-by: Michael Tremer -Signed-off-by: Peter Müller -Signed-off-by: Michael Tremer ---- - src/python/export.py | 69 ++++++++++++++++++++++++++++++++++---------- - 1 file changed, 54 insertions(+), 15 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index d15c6f0..5eaf43f 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -39,8 +39,8 @@ class OutputWriter(object): - suffix = "networks" - mode = "w" - -- def __init__(self, f, prefix=None, flatten=True): -- self.f, self.prefix, self.flatten = f, prefix, flatten -+ def __init__(self, db, f, prefix=None, flatten=True): -+ self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten - - # The previously written network - self._last_network = None -@@ -49,13 +49,13 @@ class OutputWriter(object): - self._write_header() - - @classmethod -- def open(cls, filename, **kwargs): -+ def open(cls, db, filename, **kwargs): - """ - Convenience function to open a file - """ - f = open(filename, cls.mode) - -- return cls(f, **kwargs) -+ return cls(db, f, **kwargs) - - def __repr__(self): - return "<%s f=%s>" % (self.__class__.__name__, self.f) -@@ -87,13 +87,31 @@ class OutputWriter(object): - def _write_network(self, network): - self.f.write("%s\n" % network) - -- def write(self, network): -+ def write(self, network, subnets): - if self.flatten and self._flatten(network): - log.debug("Skipping writing network %s" % network) - return - -- # Write the network to file -- self._write_network(network) -+ # Write the network when it has no subnets -+ if not subnets: -+ network = ipaddress.ip_network("%s" % network) -+ return self._write_network(network) -+ -+ # Collect all matching subnets -+ matching_subnets = [] -+ -+ for subnet in sorted(subnets): -+ # Try to find the subnet in the database -+ n = self.db.lookup("%s" % subnet.network_address) -+ -+ # No entry or matching country means those IP addresses belong here -+ if not n or n.country_code == network.country_code: -+ matching_subnets.append(subnet) -+ -+ # Write all networks as compact as possible -+ for network in ipaddress.collapse_addresses(matching_subnets): -+ log.debug("Writing %s to database" % network) -+ self._write_network(network) - - def finish(self): - """ -@@ -143,10 +161,10 @@ class XTGeoIPOutputWriter(OutputWriter): - mode = "wb" - - def _write_network(self, network): -- for address in (network.first_address, network.last_address): -+ for address in (network.network_address, network.broadcast_address): - # Convert this into a string of bits - bytes = socket.inet_pton( -- network.family, address, -+ socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, - ) - - self.f.write(bytes) -@@ -175,7 +193,7 @@ class Exporter(object): - directory, prefix=country_code, suffix=self.writer.suffix, family=family, - ) - -- writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code) -+ writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code) - - # Create writers for ASNs - for asn in asns: -@@ -183,22 +201,43 @@ class Exporter(object): - directory, "AS%s" % asn, suffix=self.writer.suffix, family=family, - ) - -- writers[asn] = self.writer.open(filename, prefix="AS%s" % asn) -+ writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) - - # Get all networks that match the family - networks = self.db.search_networks(family=family) - -+ # Materialise the generator into a list (uses quite some memory) -+ networks = list(networks) -+ - # Walk through all networks -- for network in networks: -+ for i, network in enumerate(networks): -+ _network = ipaddress.ip_network("%s" % network) -+ -+ # Search for all subnets -+ subnets = set() -+ -+ while i < len(networks): -+ subnet = networks[i+1] -+ -+ if subnet.is_subnet_of(network): -+ _subnet = ipaddress.ip_network("%s" % subnet) -+ -+ subnets.add(_subnet) -+ subnets.update(_network.address_exclude(_subnet)) -+ -+ i += 1 -+ else: -+ break -+ - # Write matching countries - try: -- writers[network.country_code].write(network) -+ writers[network.country_code].write(network, subnets) - except KeyError: - pass - - # Write matching ASNs - try: -- writers[network.asn].write(network) -+ writers[network.asn].write(network, subnets) - except KeyError: - pass - -@@ -209,7 +248,7 @@ class Exporter(object): - country = flags[flag] - - try: -- writers[country].write(network) -+ writers[country].write(network, subnets) - except KeyError: - pass - --- -2.20.1 - -From bd341642fc6bbcc050e9b4ec5124585c83cab84d Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Wed, 21 Oct 2020 14:47:39 +0000 -Subject: [PATCH 023/111] location-importer.in: filter bogus IP networks for - both Whois and extended sources -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Sanity checks for parsed networks have been put into a separate function -to avoid boilerplate code for extended sources. This makes the location -database less vulnerable to garbage written into RIR databases on -purpose or by chance. - -Fixes: #12500 - -Signed-off-by: Peter Müller -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 83 ++++++++++++++++++++++++++------- - 1 file changed, 67 insertions(+), 16 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 5656c41..f24d357 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -469,6 +469,69 @@ class CLI(object): - for line in f: - self._parse_line(line) - -+ def _check_parsed_network(self, network): -+ """ -+ Assistive function to detect and subsequently sort out parsed -+ networks from RIR data (both Whois and so-called "extended sources"), -+ which are or have... -+ -+ (a) not globally routable (RFC 1918 space, et al.) -+ (b) covering a too large chunk of the IP address space (prefix length -+ is < 7 for IPv4 networks, and < 10 for IPv6) -+ (c) "0.0.0.0" or "::" as a network address -+ (d) are too small for being publicly announced (we have decided not to -+ process them at the moment, as they significantly enlarge our -+ database without providing very helpful additional information) -+ -+ This unfortunately is necessary due to brain-dead clutter across -+ various RIR databases, causing mismatches and eventually disruptions. -+ -+ We will return False in case a network is not suitable for adding -+ it to our database, and True otherwise. -+ """ -+ -+ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)): -+ return False -+ -+ if not network.is_global: -+ logging.warning("Skipping non-globally routable network: %s" % network) -+ return False -+ -+ if network.version == 4: -+ if network.prefixlen < 7: -+ logging.warning("Skipping too big IP chunk: %s" % network) -+ return False -+ -+ if network.prefixlen > 24: -+ logging.info("Skipping network too small to be publicly announced: %s" % network) -+ return False -+ -+ if str(network.network_address) == "0.0.0.0": -+ logging.warning("Skipping network based on 0.0.0.0: %s" % network) -+ return False -+ -+ elif network.version == 6: -+ if network.prefixlen < 10: -+ logging.warning("Skipping too big IP chunk: %s" % network) -+ return False -+ -+ if network.prefixlen > 48: -+ logging.info("Skipping network too small to be publicly announced: %s" % network) -+ return False -+ -+ if str(network.network_address) == "::": -+ logging.warning("Skipping network based on '::': %s" % network) -+ return False -+ -+ else: -+ # This should not happen... -+ logging.warning("Skipping network of unknown family, this should not happen: %s" % network) -+ return False -+ -+ # In case we have made it here, the network is considered to -+ # be suitable for libloc consumption... -+ return True -+ - def _parse_block(self, block): - # Get first line to find out what type of block this is - line = block[0] -@@ -559,22 +622,7 @@ class CLI(object): - - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) - -- # Bail out in case we have processed a network covering the entire IP range, which -- # is necessary to work around faulty (?) IPv6 network processing -- if network.prefixlen == 0: -- logging.warning("Skipping network covering the entire IP adress range: %s" % network) -- return -- -- # Bail out in case we have processed a network whose prefix length indicates it is -- # not globally routable (we have decided not to process them at the moment, as they -- # significantly enlarge our database without providing very helpful additional information) -- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): -- logging.info("Skipping network too small to be publicly announced: %s" % network) -- return -- -- # Bail out in case we have processed a non-public IP network -- if network.is_private: -- logging.warning("Skipping non-globally routable network: %s" % network) -+ if not self._check_parsed_network(network): - return - - self.db.execute("INSERT INTO _rirdata(network, country) \ -@@ -658,6 +706,9 @@ class CLI(object): - log.warning("Invalid IP address: %s" % address) - return - -+ if not self._check_parsed_network(network): -+ return -+ - self.db.execute("INSERT INTO networks(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO \ - UPDATE SET country = excluded.country", --- -2.20.1 - -From eee65490a10e0fe89b3834b8be176fc900084fa0 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Wed, 21 Oct 2020 14:47:40 +0000 -Subject: [PATCH 024/111] importer.py: fetch LACNIC data via HTTPS -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Signed-off-by: Peter Müller -Signed-off-by: Michael Tremer ---- - src/python/importer.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/python/importer.py b/src/python/importer.py -index f19db4b..5f46bc3 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -64,7 +64,7 @@ EXTENDED_SOURCES = ( - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", - - # Latin America and Caribbean Network Information Centre -- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", -+ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - - # Réseaux IP Européens - #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", --- -2.20.1 - -From 84187ab5436eb158529d6f5e2a38890b4af3ddb4 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Wed, 21 Oct 2020 14:47:41 +0000 -Subject: [PATCH 025/111] location-importer.in: omit historic/orphaned RIR data -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Some RIRs include detailled information regarding networks not managed -by or allocated to themselves, particually APNIC. We need to filter -those networks (they usually have a characteristic network name) in -order to prevent operational quirks or returning wrong country codes. - -Fixes: #12501 -Partially fixes: #12499 - -Cc: Michael Tremer -Signed-off-by: Peter Müller -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 38 +++++++++++++++++++++------------ - 1 file changed, 24 insertions(+), 14 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index f24d357..a869256 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -494,38 +494,38 @@ class CLI(object): - return False - - if not network.is_global: -- logging.warning("Skipping non-globally routable network: %s" % network) -+ log.warning("Skipping non-globally routable network: %s" % network) - return False - - if network.version == 4: - if network.prefixlen < 7: -- logging.warning("Skipping too big IP chunk: %s" % network) -+ log.warning("Skipping too big IP chunk: %s" % network) - return False - - if network.prefixlen > 24: -- logging.info("Skipping network too small to be publicly announced: %s" % network) -+ log.info("Skipping network too small to be publicly announced: %s" % network) - return False - - if str(network.network_address) == "0.0.0.0": -- logging.warning("Skipping network based on 0.0.0.0: %s" % network) -+ log.warning("Skipping network based on 0.0.0.0: %s" % network) - return False - - elif network.version == 6: - if network.prefixlen < 10: -- logging.warning("Skipping too big IP chunk: %s" % network) -+ log.warning("Skipping too big IP chunk: %s" % network) - return False - - if network.prefixlen > 48: -- logging.info("Skipping network too small to be publicly announced: %s" % network) -+ log.info("Skipping network too small to be publicly announced: %s" % network) - return False - - if str(network.network_address) == "::": -- logging.warning("Skipping network based on '::': %s" % network) -+ log.warning("Skipping network based on '::': %s" % network) - return False - - else: - # This should not happen... -- logging.warning("Skipping network of unknown family, this should not happen: %s" % network) -+ log.warning("Skipping network of unknown family, this should not happen: %s" % network) - return False - - # In case we have made it here, the network is considered to -@@ -574,15 +574,22 @@ class CLI(object): - ) - - def _parse_inetnum_block(self, block): -- logging.debug("Parsing inetnum block:") -+ log.debug("Parsing inetnum block:") - - inetnum = {} - for line in block: -- logging.debug(line) -+ log.debug(line) - - # Split line - key, val = split_line(line) - -+ # Filter any inetnum records which are only referring to IP space -+ # not managed by that specific RIR... -+ if key == "netname": -+ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()): -+ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip()) -+ return -+ - if key == "inetnum": - start_address, delim, end_address = val.partition("-") - -@@ -594,7 +601,7 @@ class CLI(object): - start_address = ipaddress.ip_address(start_address) - end_address = ipaddress.ip_address(end_address) - except ValueError: -- logging.warning("Could not parse line: %s" % line) -+ log.warning("Could not parse line: %s" % line) - return - - # Set prefix to default -@@ -611,15 +618,18 @@ class CLI(object): - inetnum[key] = val - - elif key == "country": -- if val == "UNITED STATES": -- val = "US" -- - inetnum[key] = val.upper() - - # Skip empty objects - if not inetnum or not "country" in inetnum: - return - -+ # Skip objects with bogus country code 'ZZ' -+ if inetnum.get("country") == "ZZ": -+ log.warning("Skipping network with bogus country 'ZZ': %s" % \ -+ (inetnum.get("inet6num") or inetnum.get("inetnum"))) -+ return -+ - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) - - if not self._check_parsed_network(network): --- -2.20.1 - -From ebb087cfa30ec5ca0c96dcce66a91245c1ffc271 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Wed, 21 Oct 2020 14:47:43 +0000 -Subject: [PATCH 026/111] location-importer.in: avoid log spam for too small - networks -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Signed-off-by: Peter Müller -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index a869256..864eab1 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -503,7 +503,7 @@ class CLI(object): - return False - - if network.prefixlen > 24: -- log.info("Skipping network too small to be publicly announced: %s" % network) -+ log.debug("Skipping network too small to be publicly announced: %s" % network) - return False - - if str(network.network_address) == "0.0.0.0": -@@ -516,7 +516,7 @@ class CLI(object): - return False - - if network.prefixlen > 48: -- log.info("Skipping network too small to be publicly announced: %s" % network) -+ log.debug("Skipping network too small to be publicly announced: %s" % network) - return False - - if str(network.network_address) == "::": --- -2.20.1 - -From bbed1fd2330e8efa6b413dc152a1a6ef2d771aac Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 27 Oct 2020 17:14:30 +0000 -Subject: [PATCH 027/111] export: Flatten the tree before exporting it - -This patch removes the possibility that any IP address ranges -might show up in multiple exported files. - -If this was content from the database: - - * 10.0.0.0/16 - DE - * 10.0.1.0/24 - FR - -Then the IP address 10.0.1.1 would match for both countries. - -The algorithm will now break the larger /16 subnet apart into -smaller subnets so that 10.0.1.0/24 is no longer overlapped. - -There was some time spent on this to make this as efficient -as possible. - -Signed-off-by: Michael Tremer ---- - src/python/export.py | 154 ++++++++++++++++++++++++++++++------------- - 1 file changed, 110 insertions(+), 44 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index 5eaf43f..dd44332 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -89,28 +89,55 @@ class OutputWriter(object): - - def write(self, network, subnets): - if self.flatten and self._flatten(network): -- log.debug("Skipping writing network %s" % network) -+ log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network)) - return - -+ # Convert network into a Python object -+ _network = ipaddress.ip_network("%s" % network) -+ - # Write the network when it has no subnets - if not subnets: -- network = ipaddress.ip_network("%s" % network) -- return self._write_network(network) -+ log.debug("Writing %s to %s" % (_network, self.f)) -+ return self._write_network(_network) -+ -+ # Convert subnets into Python objects -+ _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets] -+ -+ # Split the network into smaller bits so that -+ # we can accomodate for any gaps in it later -+ to_check = set() -+ for _subnet in _subnets: -+ to_check.update( -+ _network.address_exclude(_subnet) -+ ) -+ -+ # Clear the list of all subnets -+ subnets = [] -+ -+ # Check if all subnets to not overlap with anything else -+ while to_check: -+ subnet_to_check = to_check.pop() - -- # Collect all matching subnets -- matching_subnets = [] -+ for _subnet in _subnets: -+ # Drop this subnet if it equals one of the subnets -+ # or if it is subnet of one of them -+ if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet): -+ break - -- for subnet in sorted(subnets): -- # Try to find the subnet in the database -- n = self.db.lookup("%s" % subnet.network_address) -+ # Break it down if it overlaps -+ if subnet_to_check.overlaps(_subnet): -+ to_check.update( -+ subnet_to_check.address_exclude(_subnet) -+ ) -+ break - -- # No entry or matching country means those IP addresses belong here -- if not n or n.country_code == network.country_code: -- matching_subnets.append(subnet) -+ # Add the subnet again as it passed the check -+ else: -+ subnets.append(subnet_to_check) - - # Write all networks as compact as possible -- for network in ipaddress.collapse_addresses(matching_subnets): -- log.debug("Writing %s to database" % network) -+ for network in ipaddress.collapse_addresses(subnets): -+ log.debug("Writing %s to %s" % (network, self.f)) - self._write_network(network) - - def finish(self): -@@ -206,40 +233,40 @@ class Exporter(object): - # Get all networks that match the family - networks = self.db.search_networks(family=family) - -- # Materialise the generator into a list (uses quite some memory) -- networks = list(networks) -+ # Create a stack with all networks in order where we can put items back -+ # again and retrieve them in the next iteration. -+ networks = BufferedStack(networks) - - # Walk through all networks -- for i, network in enumerate(networks): -- _network = ipaddress.ip_network("%s" % network) -- -- # Search for all subnets -- subnets = set() -- -- while i < len(networks): -- subnet = networks[i+1] -- -- if subnet.is_subnet_of(network): -- _subnet = ipaddress.ip_network("%s" % subnet) -- -- subnets.add(_subnet) -- subnets.update(_network.address_exclude(_subnet)) -- -- i += 1 -- else: -+ for network in networks: -+ # Collect all networks which are a subnet of network -+ subnets = [] -+ for subnet in networks: -+ # If the next subnet was not a subnet, we have to push -+ # it back on the stack and break this loop -+ if not subnet.is_subnet_of(network): -+ networks.push(subnet) - break - -+ subnets.append(subnet) -+ - # Write matching countries -- try: -- writers[network.country_code].write(network, subnets) -- except KeyError: -- pass -+ if network.country_code and network.country_code in writers: -+ # Mismatching subnets -+ gaps = [ -+ subnet for subnet in subnets if not network.country_code == subnet.country_code -+ ] -+ -+ writers[network.country_code].write(network, gaps) - - # Write matching ASNs -- try: -- writers[network.asn].write(network, subnets) -- except KeyError: -- pass -+ if network.asn and network.asn in writers: -+ # Mismatching subnets -+ gaps = [ -+ subnet for subnet in subnets if not network.asn == subnet.asn -+ ] -+ -+ writers[network.asn].write(network, gaps) - - # Handle flags - for flag in flags: -@@ -247,10 +274,19 @@ class Exporter(object): - # Fetch the "fake" country code - country = flags[flag] - -- try: -- writers[country].write(network, subnets) -- except KeyError: -- pass -+ if not country in writers: -+ continue -+ -+ gaps = [ -+ subnet for subnet in subnets -+ if not subnet.has_flag(flag) -+ ] -+ -+ writers[country].write(network, gaps) -+ -+ # Push all subnets back onto the stack -+ for subnet in reversed(subnets): -+ networks.push(subnet) - - # Write everything to the filesystem - for writer in writers.values(): -@@ -262,3 +298,33 @@ class Exporter(object): - ) - - return os.path.join(directory, filename) -+ -+ -+class BufferedStack(object): -+ """ -+ This class takes an iterator and when being iterated -+ over it returns objects from that iterator for as long -+ as there are any. -+ -+ It additionally has a function to put an item back on -+ the back so that it will be returned again at the next -+ iteration. -+ """ -+ def __init__(self, iterator): -+ self.iterator = iterator -+ self.stack = [] -+ -+ def __iter__(self): -+ return self -+ -+ def __next__(self): -+ if self.stack: -+ return self.stack.pop(0) -+ -+ return next(self.iterator) -+ -+ def push(self, elem): -+ """ -+ Takes an element and puts it on the stack -+ """ -+ self.stack.insert(0, elem) --- -2.20.1 - -From e99a72265c1ba2194b61663eda7e9f14e0083016 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 28 Oct 2020 09:52:36 +0000 -Subject: [PATCH 028/111] location: Fix Python syntax error in verify() - -The database is now being opened before the requested -method is called and handle_verify() wasn't updated. - -Signed-off-by: Michael Tremer ---- - src/python/location.in | 8 +------- - 1 file changed, 1 insertion(+), 7 deletions(-) - -diff --git a/src/python/location.in b/src/python/location.in -index 44ad726..b5e5758 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -453,13 +453,7 @@ class CLI(object): - - return 0 - -- def handle_verify(self, ns): -- try: -- db = location.Database(ns.database) -- except FileNotFoundError as e: -- log.error("%s: %s" % (ns.database, e)) -- return 127 -- -+ def handle_verify(self, db, ns): - # Verify the database - with open(ns.public_key, "r") as f: - if not db.verify(f): --- -2.20.1 - -From 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Thu, 29 Oct 2020 07:25:53 -0700 -Subject: [PATCH 029/111] location update: Remove double conversion of - timestamps -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This caused that the timestamp in DNS was misinterpreted -as local time and often, databases could not be downloaded. - -Signed-off-by: Peter Müller ---- - src/python/location.in | 5 +---- - 1 file changed, 1 insertion(+), 4 deletions(-) - -diff --git a/src/python/location.in b/src/python/location.in -index b5e5758..070640c 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -421,11 +421,8 @@ class CLI(object): - # Fetch the timestamp we need from DNS - t = location.discover_latest_version() - -- # Parse timestamp into datetime format -- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None -- - # Check the version of the local database -- if db and timestamp and db.created_at >= timestamp.timestamp(): -+ if db and t and db.created_at >= t: - log.info("Already on the latest version") - return - --- -2.20.1 - -From 60c1ac0307312614bd6980d30b44bb59b5a6ab6e Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Thu, 29 Oct 2020 07:36:46 -0700 -Subject: [PATCH 030/111] location.in: do not confuse UTC with local time zones -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Signed-off-by: Peter Müller ---- - src/python/location.in | 12 +++++------- - 1 file changed, 5 insertions(+), 7 deletions(-) - -diff --git a/src/python/location.in b/src/python/location.in -index 070640c..0d09210 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -398,10 +398,7 @@ class CLI(object): - - def handle_update(self, db, ns): - if ns.cron and db: -- now = datetime.datetime.utcnow() -- -- # Parse the database timestamp -- t = datetime.datetime.utcfromtimestamp(db.created_at) -+ now = time.time() - - if ns.cron == "daily": - delta = datetime.timedelta(days=1) -@@ -410,11 +407,12 @@ class CLI(object): - elif ns.cron == "monthly": - delta = datetime.timedelta(days=30) - -+ delta = delta.total_seconds() -+ - # Check if the database has recently been updated -- if t >= (now - delta): -+ if db.created_at >= (now - delta): - log.info( -- _("The database has been updated recently (%s)") % \ -- format_timedelta(now - t), -+ _("The database has been updated recently"), - ) - return 3 - --- -2.20.1 - -From e7d612e5219ef9ba612ed404e4e2c174110d3dd7 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= -Date: Tue, 3 Nov 2020 15:31:08 +0000 -Subject: [PATCH 031/111] location-importer.in: always convert organisation - handles into upper cases -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Fixes: #12523 - -Signed-off-by: Peter Müller -Signed-off-by: Michael Tremer ---- - src/python/location-importer.in | 6 ++++-- - 1 file changed, 4 insertions(+), 2 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 864eab1..2dec89e 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -560,7 +560,7 @@ class CLI(object): - autnum["asn"] = m.group(2) - - elif key == "org": -- autnum[key] = val -+ autnum[key] = val.upper() - - # Skip empty objects - if not autnum: -@@ -646,7 +646,9 @@ class CLI(object): - # Split line - key, val = split_line(line) - -- if key in ("organisation", "org-name"): -+ if key == "organisation": -+ org[key] = val.upper() -+ elif key == "org-name": - org[key] = val - - # Skip empty objects --- -2.20.1 - -From e96704f43acca1a8f56d9a680cce281f5e587ec5 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 11 Nov 2020 21:16:45 +0000 -Subject: [PATCH 032/111] test: Add tests for database enumerator - -Signed-off-by: Michael Tremer ---- - src/test-database.c | 53 +++++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 53 insertions(+) - -diff --git a/src/test-database.c b/src/test-database.c -index b4a75c4..4aef94e 100644 ---- a/src/test-database.c -+++ b/src/test-database.c -@@ -38,6 +38,14 @@ const char* DESCRIPTION = - "Maecenas ut venenatis nunc."; - const char* LICENSE = "CC"; - -+const char* networks[] = { -+ "2001:db8::/32", -+ "2001:db8:1000::/48", -+ "2001:db8:2000::/48", -+ "2001:db8:2020::/48", -+ NULL, -+}; -+ - static int attempt_to_open(struct loc_ctx* ctx, char* path) { - FILE* f = fopen(path, "r"); - if (!f) -@@ -139,6 +147,24 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ struct loc_network* network = NULL; -+ -+ // Add some networks -+ const char** n = networks; -+ while (*n) { -+ err = loc_writer_add_network(writer, &network, *n); -+ if (err) { -+ fprintf(stderr, "Could not add network %s\n", *n); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Set a country -+ loc_network_set_country_code(network, "XX"); -+ -+ // Next one -+ n++; -+ } -+ - FILE* f = tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); -@@ -170,6 +196,33 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ // Enumerator -+ struct loc_database_enumerator* enumerator; -+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS); -+ if (err) { -+ fprintf(stderr, "Could not initialise the enumerator: %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Walk through all networks -+ while (1) { -+ err = loc_database_enumerator_next_network(enumerator, &network); -+ if (err) { -+ fprintf(stderr, "Error fetching the next network: %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (!network) -+ break; -+ -+ char* s = loc_network_str(network); -+ printf("Got network: %s\n", s); -+ free(s); -+ } -+ -+ // Free the enumerator -+ loc_database_enumerator_unref(enumerator); -+ - // Close the database - loc_database_unref(db); - loc_unref(ctx); --- -2.20.1 - -From ecce288da39a2c0eb60076050ca21e9619f61844 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 11 Nov 2020 23:01:19 +0000 -Subject: [PATCH 033/111] networks: Add list to manage groups of networks - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 11 ++++++ - src/loc/network.h | 12 ++++++ - src/network.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++ - 3 files changed, 119 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index b8296eb..6ef2d27 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -98,6 +98,17 @@ global: - loc_network_str; - loc_network_unref; - -+ # Network List -+ loc_network_list_clear; -+ loc_network_list_empty; -+ loc_network_list_get; -+ loc_network_list_new; -+ loc_network_list_pop; -+ loc_network_list_push; -+ loc_network_list_ref; -+ loc_network_list_size; -+ loc_network_list_unref; -+ - # Writer - loc_writer_add_as; - loc_writer_add_country; -diff --git a/src/loc/network.h b/src/loc/network.h -index 70c3803..fd20812 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -56,6 +56,18 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); - -+// List -+struct loc_network_list; -+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); -+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); -+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); -+size_t loc_network_list_size(struct loc_network_list* list); -+int loc_network_list_empty(struct loc_network_list* list); -+void loc_network_list_clear(struct loc_network_list* list); -+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); -+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); -+struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+ - #ifdef LIBLOC_PRIVATE - - int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj); -diff --git a/src/network.c b/src/network.c -index d7b1645..c9e7979 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -746,3 +746,99 @@ LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) - LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { - return loc_network_ref(node->network); - } -+ -+// List -+ -+struct loc_network_list { -+ struct loc_ctx* ctx; -+ int refcount; -+ -+ struct loc_network* list[1024]; -+ size_t size; -+ size_t max_size; -+}; -+ -+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, -+ struct loc_network_list** list) { -+ struct loc_network_list* l = calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; -+ -+ l->ctx = loc_ref(ctx); -+ l->refcount = 1; -+ -+ // Do not allow this list to grow larger than this -+ l->max_size = 1024; -+ -+ DEBUG(l->ctx, "Network list allocated at %p\n", l); -+ *list = l; -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { -+ list->refcount++; -+ -+ return list; -+} -+ -+static void loc_network_list_free(struct loc_network_list* list) { -+ DEBUG(list->ctx, "Releasing network list at %p\n", list); -+ -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_network_unref(list->list[i]); -+ -+ loc_unref(list->ctx); -+ free(list); -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { -+ if (!list) -+ return NULL; -+ -+ if (--list->refcount > 0) -+ return list; -+ -+ loc_network_list_free(list); -+ return NULL; -+} -+ -+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { -+ return list->size; -+} -+ -+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { -+ return list->size == 0; -+} -+ -+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_network_unref(list->list[i]); -+ -+ list->size = 0; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { -+ // Check index -+ if (index >= list->size) -+ return NULL; -+ -+ return loc_network_ref(list->list[index]); -+} -+ -+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -+ // Check if we have space left -+ if (list->size == list->max_size) -+ return -ENOMEM; -+ -+ list->list[list->size++] = loc_network_ref(network); -+ -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) -+ return NULL; -+ -+ return list->list[list->size--]; -+} --- -2.20.1 - -From 8b2205272b7872a1458ad87811abf58609f38ad4 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 13:57:35 +0000 -Subject: [PATCH 034/111] networks: Add function to dump lists - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 14 ++++++++++++++ - 3 files changed, 16 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 6ef2d27..a5641c6 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -100,6 +100,7 @@ global: - - # Network List - loc_network_list_clear; -+ loc_network_list_dump; - loc_network_list_empty; - loc_network_list_get; - loc_network_list_new; -diff --git a/src/loc/network.h b/src/loc/network.h -index fd20812..44c50a4 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -64,6 +64,7 @@ struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); - size_t loc_network_list_size(struct loc_network_list* list); - int loc_network_list_empty(struct loc_network_list* list); - void loc_network_list_clear(struct loc_network_list* list); -+void loc_network_list_dump(struct loc_network_list* list); - struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); - int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); - struct loc_network* loc_network_list_pop(struct loc_network_list* list); -diff --git a/src/network.c b/src/network.c -index c9e7979..0977406 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -817,6 +817,20 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { - list->size = 0; - } - -+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { -+ struct loc_network* network; -+ char* s; -+ -+ for (unsigned int i = 0; i < list->size; i++) { -+ network = list->list[i]; -+ -+ s = loc_network_str(network); -+ -+ INFO(list->ctx, "%s\n", s); -+ free(s); -+ } -+} -+ - LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { - // Check index - if (index >= list->size) --- -2.20.1 - -From 850e75167e8e03fe8b951992c9f7bc2ccb9fb711 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 14:18:40 +0000 -Subject: [PATCH 035/111] network: Add functions to break network into subnets - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 5 + - src/loc/network.h | 6 + - src/network.c | 290 ++++++++++++++++++++++++++++++++++++++++++++- - src/test-network.c | 61 ++++++++++ - 4 files changed, 361 insertions(+), 1 deletion(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index a5641c6..fcb9ea5 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -80,6 +80,8 @@ global: - - # Network - loc_network_address_family; -+ loc_network_eq; -+ loc_network_exclude; - loc_network_format_first_address; - loc_network_format_last_address; - loc_network_get_asn; -@@ -96,6 +98,7 @@ global: - loc_network_set_country_code; - loc_network_set_flag; - loc_network_str; -+ loc_network_subnets; - loc_network_unref; - - # Network List -@@ -107,7 +110,9 @@ global: - loc_network_list_pop; - loc_network_list_push; - loc_network_list_ref; -+ loc_network_list_reverse; - loc_network_list_size; -+ loc_network_list_sort; - loc_network_list_unref; - - # Writer -diff --git a/src/loc/network.h b/src/loc/network.h -index 44c50a4..4e51a62 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -54,7 +54,11 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag); - int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); - -+int loc_network_eq(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); -+struct loc_network_list* loc_network_subnets(struct loc_network* network); -+struct loc_network_list* loc_network_exclude( -+ struct loc_network* self, struct loc_network* other); - - // List - struct loc_network_list; -@@ -68,6 +72,8 @@ void loc_network_list_dump(struct loc_network_list* list); - struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); - int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); - struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+void loc_network_list_sort(struct loc_network_list* list); -+void loc_network_list_reverse(struct loc_network_list* list); - - #ifdef LIBLOC_PRIVATE - -diff --git a/src/network.c b/src/network.c -index 0977406..6c08070 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -97,6 +97,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s - return a; - } - -+static struct in6_addr address_increment(const struct in6_addr* address) { -+ struct in6_addr a = *address; -+ -+ for (int octet = 15; octet >= 0; octet--) { -+ if (a.s6_addr[octet] < 255) { -+ a.s6_addr[octet]++; -+ break; -+ } else { -+ a.s6_addr[octet] = 0; -+ } -+ } -+ -+ return a; -+} -+ - LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network, - struct in6_addr* address, unsigned int prefix) { - // Address cannot be unspecified -@@ -405,6 +420,69 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag - return loc_network_has_flag(network, flag); - } - -+LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { -+#ifdef ENABLE_DEBUG -+ char* n1 = loc_network_str(self); -+ char* n2 = loc_network_str(other); -+ -+ DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2); -+ -+ free(n1); -+ free(n2); -+#endif -+ -+ // Family must be the same -+ if (self->family != other->family) { -+ DEBUG(self->ctx, " Family mismatch\n"); -+ -+ return 0; -+ } -+ -+ // The start address must be the same -+ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) { -+ DEBUG(self->ctx, " Address mismatch\n"); -+ -+ return 0; -+ } -+ -+ // The prefix length must be the same -+ if (self->prefix != other->prefix) { -+ DEBUG(self->ctx, " Prefix mismatch\n"); -+ return 0; -+ } -+ -+ DEBUG(self->ctx, " Yes!\n"); -+ -+ return 1; -+} -+ -+static int loc_network_gt(struct loc_network* self, struct loc_network* other) { -+ // Families must match -+ if (self->family != other->family) -+ return -1; -+ -+ int r = in6_addr_cmp(&self->first_address, &other->first_address); -+ -+ switch (r) { -+ // Smaller -+ case -1: -+ return 0; -+ -+ // Larger -+ case 1: -+ return 1; -+ -+ default: -+ break; -+ } -+ -+ if (self->prefix > other->prefix) -+ return 1; -+ -+ // Dunno -+ return 0; -+} -+ - LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. -@@ -419,6 +497,175 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net - return 1; - } - -+LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) { -+ struct loc_network_list* list; -+ -+ // New prefix length -+ unsigned int prefix = network->prefix + 1; -+ -+ // Check if the new prefix is valid -+ if (valid_prefix(&network->first_address, prefix)) -+ return NULL; -+ -+ // Create a new list with the result -+ int r = loc_network_list_new(network->ctx, &list); -+ if (r) { -+ ERROR(network->ctx, "Could not create network list: %d\n", r); -+ return NULL; -+ } -+ -+ struct loc_network* subnet1 = NULL; -+ struct loc_network* subnet2 = NULL; -+ -+ // Create the first half of the network -+ r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix); -+ if (r) -+ goto ERROR; -+ -+ // The next subnet starts after the first one -+ struct in6_addr first_address = address_increment(&subnet1->last_address); -+ -+ // Create the second half of the network -+ r = loc_network_new(network->ctx, &subnet2, &first_address, prefix); -+ if (r) -+ goto ERROR; -+ -+ // Push the both onto the stack (in reverse order) -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; -+ -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; -+ -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); -+ -+ return list; -+ -+ERROR: -+ if (subnet1) -+ loc_network_unref(subnet1); -+ -+ if (subnet2) -+ loc_network_unref(subnet2); -+ -+ if (list) -+ loc_network_list_unref(list); -+ -+ return NULL; -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_exclude( -+ struct loc_network* self, struct loc_network* other) { -+ struct loc_network_list* list; -+ -+#ifdef ENABLE_DEBUG -+ char* n1 = loc_network_str(self); -+ char* n2 = loc_network_str(other); -+ -+ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); -+ -+ free(n1); -+ free(n2); -+#endif -+ -+ // Family must match -+ if (self->family != other->family) { -+ DEBUG(self->ctx, "Family mismatch\n"); -+ -+ return NULL; -+ } -+ -+ // Other must be a subnet of self -+ if (!loc_network_is_subnet_of(other, self)) { -+ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); -+ -+ return NULL; -+ } -+ -+ // We cannot perform this operation if both networks equal -+ if (loc_network_eq(self, other)) { -+ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); -+ -+ return NULL; -+ } -+ -+ // Create a new list with the result -+ int r = loc_network_list_new(self->ctx, &list); -+ if (r) { -+ ERROR(self->ctx, "Could not create network list: %d\n", r); -+ return NULL; -+ } -+ -+ struct loc_network_list* subnets = loc_network_subnets(self); -+ -+ struct loc_network* subnet1 = NULL; -+ struct loc_network* subnet2 = NULL; -+ -+ while (subnets) { -+ // Fetch both subnets -+ subnet1 = loc_network_list_get(subnets, 0); -+ subnet2 = loc_network_list_get(subnets, 1); -+ -+ // Free list -+ loc_network_list_unref(subnets); -+ subnets = NULL; -+ -+ if (loc_network_eq(other, subnet1)) { -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; -+ -+ } else if (loc_network_eq(other, subnet2)) { -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; -+ -+ } else if (loc_network_is_subnet_of(other, subnet1)) { -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; -+ -+ subnets = loc_network_subnets(subnet1); -+ -+ } else if (loc_network_is_subnet_of(other, subnet2)) { -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; -+ -+ subnets = loc_network_subnets(subnet2); -+ -+ } else { -+ ERROR(self->ctx, "We should never get here\n"); -+ goto ERROR; -+ } -+ -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); -+ } -+ -+#ifdef ENABLE_DEBUG -+ loc_network_list_dump(list); -+#endif -+ -+ // Return the result -+ return list; -+ -+ERROR: -+ if (subnet1) -+ loc_network_unref(subnet1); -+ -+ if (subnet2) -+ loc_network_unref(subnet2); -+ -+ if (list) -+ loc_network_list_unref(list); -+ -+ return NULL; -+} -+ - LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { - // Add country code - loc_country_code_copy(dbobj->country_code, network->country_code); -@@ -854,5 +1101,46 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis - if (loc_network_list_empty(list)) - return NULL; - -- return list->list[list->size--]; -+ return list->list[--list->size]; -+} -+ -+static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { -+ // Do nothing for invalid indices -+ if (i1 >= list->size || i2 >= list->size) -+ return; -+ -+ DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2); -+ -+ struct loc_network* network1 = list->list[i1]; -+ struct loc_network* network2 = list->list[i2]; -+ -+ list->list[i1] = network2; -+ list->list[i2] = network1; -+} -+ -+LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { -+ unsigned int i = 0; -+ unsigned int j = list->size - 1; -+ -+ while (i < j) { -+ loc_network_list_swap(list, i++, j--); -+ } -+} -+ -+LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { -+ unsigned int n = list->size; -+ int swapped; -+ -+ do { -+ swapped = 0; -+ -+ for (unsigned int i = 1; i < n; i++) { -+ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { -+ loc_network_list_swap(list, i-1, i); -+ swapped = 1; -+ } -+ } -+ -+ n--; -+ } while (swapped); - } -diff --git a/src/test-network.c b/src/test-network.c -index b6776b4..af1b2e6 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -118,6 +118,19 @@ int main(int argc, char** argv) { - size_t nodes = loc_network_tree_count_nodes(tree); - printf("The tree has %zu nodes\n", nodes); - -+ // Check equals function -+ err = loc_network_eq(network1, network1); -+ if (!err) { -+ fprintf(stderr, "Network is not equal with itself\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ err = loc_network_eq(network1, network2); -+ if (err) { -+ fprintf(stderr, "Networks equal unexpectedly\n"); -+ exit(EXIT_FAILURE); -+ } -+ - // Check subnet function - err = loc_network_is_subnet_of(network1, network2); - if (err != 0) { -@@ -131,6 +144,54 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ // Make list of subnets -+ struct loc_network_list* subnets = loc_network_subnets(network1); -+ if (!subnets) { -+ fprintf(stderr, "Could not find subnets of network\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(subnets); -+ -+ while (!loc_network_list_empty(subnets)) { -+ struct loc_network* subnet = loc_network_list_pop(subnets); -+ if (!subnet) { -+ fprintf(stderr, "Received an empty subnet\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ char* s = loc_network_str(subnet); -+ printf("Received subnet %s\n", s); -+ free(s); -+ -+ if (!loc_network_is_subnet_of(subnet, network1)) { -+ fprintf(stderr, "Not a subnet\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_unref(subnet); -+ } -+ -+ loc_network_list_unref(subnets); -+ -+ struct loc_network_list* excluded = loc_network_exclude(network1, network2); -+ if (!excluded) { -+ fprintf(stderr, "Could not create excluded list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ -+ // Reverse it -+ loc_network_list_reverse(excluded); -+ loc_network_list_dump(excluded); -+ -+ // Sort them and dump them again -+ loc_network_list_sort(excluded); -+ loc_network_list_dump(excluded); -+ -+ loc_network_list_unref(excluded); -+ - // Create a database - struct loc_writer* writer; - err = loc_writer_new(ctx, &writer, NULL, NULL); --- -2.20.1 - -From 6159d384c4a98fe45ec52522e2950719e4982d80 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 14:24:58 +0000 -Subject: [PATCH 036/111] networks: Add function to check if two networks - overlap - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 16 ++++++++++++++++ - 3 files changed, 18 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index fcb9ea5..0c2835b 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -93,6 +93,7 @@ global: - loc_network_match_flag; - loc_network_new; - loc_network_new_from_string; -+ loc_network_overlaps; - loc_network_ref; - loc_network_set_asn; - loc_network_set_country_code; -diff --git a/src/loc/network.h b/src/loc/network.h -index 4e51a62..ef13756 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -55,6 +55,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_eq(struct loc_network* self, struct loc_network* other); -+int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); - struct loc_network_list* loc_network_subnets(struct loc_network* network); - struct loc_network_list* loc_network_exclude( -diff --git a/src/network.c b/src/network.c -index 6c08070..d826511 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -483,6 +483,22 @@ static int loc_network_gt(struct loc_network* self, struct loc_network* other) { - return 0; - } - -+LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { -+ if (loc_network_match_address(self, &other->first_address) == 0) -+ return 1; -+ -+ if (loc_network_match_address(self, &other->last_address) == 0) -+ return 1; -+ -+ if (loc_network_match_address(other, &self->first_address) == 0) -+ return 1; -+ -+ if (loc_network_match_address(other, &self->last_address) == 0) -+ return 1; -+ -+ return 0; -+} -+ - LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. --- -2.20.1 - -From e52ba21761f27e592040a2793b2a26bbeeeecc05 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 14:28:15 +0000 -Subject: [PATCH 037/111] networks: Add function to check if network is part of - a list - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 9 +++++++++ - 3 files changed, 11 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 0c2835b..f1b63a2 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -104,6 +104,7 @@ global: - - # Network List - loc_network_list_clear; -+ loc_network_list_contains; - loc_network_list_dump; - loc_network_list_empty; - loc_network_list_get; -diff --git a/src/loc/network.h b/src/loc/network.h -index ef13756..7804512 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -73,6 +73,7 @@ void loc_network_list_dump(struct loc_network_list* list); - struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); - int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); - struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); - void loc_network_list_sort(struct loc_network_list* list); - void loc_network_list_reverse(struct loc_network_list* list); - -diff --git a/src/network.c b/src/network.c -index d826511..fcbdc59 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -1120,6 +1120,15 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis - return list->list[--list->size]; - } - -+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { -+ for (unsigned int i = 0; i < list->size; i++) { -+ if (loc_network_eq(list->list[i], network)) -+ return 1; -+ } -+ -+ return 0; -+} -+ - static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { - // Do nothing for invalid indices - if (i1 >= list->size || i2 >= list->size) --- -2.20.1 - -From f802f3a4decf4827ecc8bcabe269ae9f94f7f32d Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 14:33:22 +0000 -Subject: [PATCH 038/111] networks: Add function to merge two lists - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 13 +++++++++++++ - 3 files changed, 15 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index f1b63a2..c0b6b1f 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -108,6 +108,7 @@ global: - loc_network_list_dump; - loc_network_list_empty; - loc_network_list_get; -+ loc_network_list_merge; - loc_network_list_new; - loc_network_list_pop; - loc_network_list_push; -diff --git a/src/loc/network.h b/src/loc/network.h -index 7804512..e30d91c 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -76,6 +76,7 @@ struct loc_network* loc_network_list_pop(struct loc_network_list* list); - int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); - void loc_network_list_sort(struct loc_network_list* list); - void loc_network_list_reverse(struct loc_network_list* list); -+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); - - #ifdef LIBLOC_PRIVATE - -diff --git a/src/network.c b/src/network.c -index fcbdc59..541286d 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -1169,3 +1169,16 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { - n--; - } while (swapped); - } -+ -+LOC_EXPORT int loc_network_list_merge( -+ struct loc_network_list* self, struct loc_network_list* other) { -+ int r; -+ -+ for (unsigned int i = 0; i < other->size; i++) { -+ r = loc_network_list_push(self, other->list[i]); -+ if (r) -+ return r; -+ } -+ -+ return 0; -+} --- -2.20.1 - -From 6d22a179dffd08fcf2a44aafb361725ab22486d4 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 14:35:43 +0000 -Subject: [PATCH 039/111] network: Make lists unique - -Networks that are in the list won't be added again - -Signed-off-by: Michael Tremer ---- - src/network.c | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/src/network.c b/src/network.c -index 541286d..44571b3 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -1103,6 +1103,10 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis - } - - LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -+ // Do not add networks that are already on the list -+ if (loc_network_list_contains(list, network)) -+ return 0; -+ - // Check if we have space left - if (list->size == list->max_size) - return -ENOMEM; --- -2.20.1 - -From 681ff05cb7cdf230d38abf09a330a31498e265a4 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 19:21:13 +0000 -Subject: [PATCH 040/111] database: Pass flag to enumerator to flatten output - -Signed-off-by: Michael Tremer ---- - src/database.c | 8 +++++++- - src/loc/database.h | 6 +++++- - src/perl/Location.xs | 2 +- - src/python/database.c | 25 ++++++++++++++++++------- - 4 files changed, 31 insertions(+), 10 deletions(-) - -diff --git a/src/database.c b/src/database.c -index fa1dad0..9baab33 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -104,6 +104,9 @@ struct loc_database_enumerator { - enum loc_network_flags flags; - int family; - -+ // Flatten output? -+ int flatten; -+ - // Index of the AS we are looking at - unsigned int as_index; - -@@ -933,7 +936,7 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, - // Enumerator - - LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, -- struct loc_database* db, enum loc_database_enumerator_mode mode) { -+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { - struct loc_database_enumerator* e = calloc(1, sizeof(*e)); - if (!e) - return -ENOMEM; -@@ -944,6 +947,9 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum - e->mode = mode; - e->refcount = 1; - -+ // Flatten output? -+ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); -+ - // Initialise graph search - //e->network_stack[++e->network_stack_depth] = 0; - e->network_stack_depth = 1; -diff --git a/src/loc/database.h b/src/loc/database.h -index 43173dd..14eb5ea 100644 ---- a/src/loc/database.h -+++ b/src/loc/database.h -@@ -55,9 +55,13 @@ enum loc_database_enumerator_mode { - LOC_DB_ENUMERATE_COUNTRIES = 3, - }; - -+enum loc_database_enumerator_flags { -+ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0), -+}; -+ - struct loc_database_enumerator; - int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, -- struct loc_database* db, enum loc_database_enumerator_mode mode); -+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags); - struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator); - struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); - -diff --git a/src/perl/Location.xs b/src/perl/Location.xs -index dcf3f0d..b7676d2 100644 ---- a/src/perl/Location.xs -+++ b/src/perl/Location.xs -@@ -125,7 +125,7 @@ database_countries(db) - PPCODE: - // Create Database enumerator - struct loc_database_enumerator* enumerator; -- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES); -+ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0); - - if (err) { - croak("Could not create a database enumerator\n"); -diff --git a/src/python/database.c b/src/python/database.c -index 1013a58..7f8c2c2 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -207,10 +207,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database - return (PyObject*)self; - } - --static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) { -+static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) { - struct loc_database_enumerator* enumerator; - -- int r = loc_database_enumerator_new(&enumerator, self->db, what); -+ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -@@ -223,7 +223,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en - } - - static PyObject* Database_ases(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0); - } - - static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { -@@ -234,7 +234,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { - - struct loc_database_enumerator* enumerator; - -- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES); -+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -@@ -250,7 +250,11 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { - } - - static PyObject* Database_networks(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0); -+} -+ -+static PyObject* Database_networks_flattened(DatabaseObject *self) { -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN); - } - - static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { -@@ -264,7 +268,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - return NULL; - - struct loc_database_enumerator* enumerator; -- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS); -+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -@@ -317,7 +321,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - } - - static PyObject* Database_countries(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0); - } - - static struct PyMethodDef Database_methods[] = { -@@ -403,6 +407,13 @@ static struct PyGetSetDef Database_getsetters[] = { - NULL, - NULL, - }, -+ { -+ "networks_flattened", -+ (getter)Database_networks_flattened, -+ NULL, -+ NULL, -+ NULL, -+ }, - { - "vendor", - (getter)Database_get_vendor, --- -2.20.1 - -From f5e50a47e37e9b29d0d2ee9e5a41e5a5fe5aea7f Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 19:21:58 +0000 -Subject: [PATCH 041/111] network: Reduce debugging output - -Signed-off-by: Michael Tremer ---- - src/network.c | 28 +++------------------------- - 1 file changed, 3 insertions(+), 25 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 44571b3..f7071a6 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -421,37 +421,17 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag - } - - LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { --#ifdef ENABLE_DEBUG -- char* n1 = loc_network_str(self); -- char* n2 = loc_network_str(other); -- -- DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2); -- -- free(n1); -- free(n2); --#endif -- - // Family must be the same -- if (self->family != other->family) { -- DEBUG(self->ctx, " Family mismatch\n"); -- -+ if (self->family != other->family) - return 0; -- } - - // The start address must be the same -- if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) { -- DEBUG(self->ctx, " Address mismatch\n"); -- -+ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) - return 0; -- } - - // The prefix length must be the same -- if (self->prefix != other->prefix) { -- DEBUG(self->ctx, " Prefix mismatch\n"); -+ if (self->prefix != other->prefix) - return 0; -- } -- -- DEBUG(self->ctx, " Yes!\n"); - - return 1; - } -@@ -1138,8 +1118,6 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1 - if (i1 >= list->size || i2 >= list->size) - return; - -- DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2); -- - struct loc_network* network1 = list->list[i1]; - struct loc_network* network2 = list->list[i2]; - --- -2.20.1 - -From 037c65d3a07ec6d37ff063f0645adda6b483b407 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 19:36:38 +0000 -Subject: [PATCH 042/111] python: Export networks exclude function - -Signed-off-by: Michael Tremer ---- - src/python/network.c | 39 +++++++++++++++++++++++++++++++++++++++ - 1 file changed, 39 insertions(+) - -diff --git a/src/python/network.c b/src/python/network.c -index 5496d1e..11f672b 100644 ---- a/src/python/network.c -+++ b/src/python/network.c -@@ -24,6 +24,24 @@ - #include "locationmodule.h" - #include "network.h" - -+static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) { -+ PyObject* list = PyList_New(0); -+ if (!networks) -+ return list; -+ -+ while (!loc_network_list_empty(networks)) { -+ struct loc_network* network = loc_network_list_pop(networks); -+ -+ PyObject* n = new_network(&NetworkType, network); -+ PyList_Append(list, n); -+ -+ loc_network_unref(network); -+ Py_DECREF(n); -+ } -+ -+ return list; -+} -+ - PyObject* new_network(PyTypeObject* type, struct loc_network* network) { - NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0); - if (self) { -@@ -154,6 +172,21 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) { - Py_RETURN_NONE; - } - -+static PyObject* Network_exclude(NetworkObject* self, PyObject* args) { -+ NetworkObject* other = NULL; -+ -+ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) -+ return NULL; -+ -+ struct loc_network_list* list = loc_network_exclude(self->network, other->network); -+ -+ // Convert to Python objects -+ PyObject* obj = PyList_FromNetworkList(list); -+ loc_network_list_unref(list); -+ -+ return obj; -+} -+ - static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { - NetworkObject* other = NULL; - -@@ -191,6 +224,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) { - } - - static struct PyMethodDef Network_methods[] = { -+ { -+ "exclude", -+ (PyCFunction)Network_exclude, -+ METH_VARARGS, -+ NULL, -+ }, - { - "has_flag", - (PyCFunction)Network_has_flag, --- -2.20.1 - -From 9a7732c8679e805d4d2d55ea4750c5d70ca4bd2c Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 19:59:22 +0000 -Subject: [PATCH 043/111] network: Add more debugging output to stacks - -Signed-off-by: Michael Tremer ---- - src/network.c | 16 +++++++++++++--- - 1 file changed, 13 insertions(+), 3 deletions(-) - -diff --git a/src/network.c b/src/network.c -index f7071a6..d41e873 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -1088,8 +1088,12 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n - return 0; - - // Check if we have space left -- if (list->size == list->max_size) -+ if (list->size == list->max_size) { -+ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); - return -ENOMEM; -+ } -+ -+ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); - - list->list[list->size++] = loc_network_ref(network); - -@@ -1098,10 +1102,16 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n - - LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { - // Return nothing when empty -- if (loc_network_list_empty(list)) -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); - return NULL; -+ } - -- return list->list[--list->size]; -+ struct loc_network* network = list->list[--list->size]; -+ -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -+ -+ return network; - } - - LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { --- -2.20.1 - -From 33a051e0435f6e78cc936f26f3b9ee16b7851025 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 20:00:09 +0000 -Subject: [PATCH 044/111] network: Add new subnet function - -The old one is too difficult to use in terms of order -of input parameters and return value. - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 15 +++++++++++++++ - 3 files changed, 17 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index c0b6b1f..5392437 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -87,6 +87,7 @@ global: - loc_network_get_asn; - loc_network_get_country_code; - loc_network_has_flag; -+ loc_network_is_subnet; - loc_network_is_subnet_of; - loc_network_match_asn; - loc_network_match_country_code; -diff --git a/src/loc/network.h b/src/loc/network.h -index e30d91c..2154cdc 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -56,6 +56,7 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_eq(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); -+int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); - struct loc_network_list* loc_network_subnets(struct loc_network* network); - struct loc_network_list* loc_network_exclude( -diff --git a/src/network.c b/src/network.c -index d41e873..5719111 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -479,6 +479,21 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network - return 0; - } - -+LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { -+ // If the start address of the other network is smaller than this network, -+ // it cannot be a subnet. -+ if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) -+ return 0; -+ -+ // If the end address of the other network is greater than this network, -+ // it cannot be a subnet. -+ if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) -+ return 0; -+ -+ return 1; -+} -+ -+// XXX DEPRECATED - I find this too difficult to use - LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. --- -2.20.1 - -From add5bb652ba1dad1127f79cb6a0db2d363a6d5e5 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 20:01:17 +0000 -Subject: [PATCH 045/111] network: Add function to exclude multiple networks at - once - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 1 + - src/loc/network.h | 2 ++ - src/network.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++ - 3 files changed, 88 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 5392437..bcd11be 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -82,6 +82,7 @@ global: - loc_network_address_family; - loc_network_eq; - loc_network_exclude; -+ loc_network_exclude_list; - loc_network_format_first_address; - loc_network_format_last_address; - loc_network_get_asn; -diff --git a/src/loc/network.h b/src/loc/network.h -index 2154cdc..40712b9 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -61,6 +61,8 @@ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other - struct loc_network_list* loc_network_subnets(struct loc_network* network); - struct loc_network_list* loc_network_exclude( - struct loc_network* self, struct loc_network* other); -+struct loc_network_list* loc_network_exclude_list( -+ struct loc_network* network, struct loc_network_list* list); - - // List - struct loc_network_list; -diff --git a/src/network.c b/src/network.c -index 5719111..751e8e5 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -677,6 +677,91 @@ ERROR: - return NULL; - } - -+LOC_EXPORT struct loc_network_list* loc_network_exclude_list( -+ struct loc_network* network, struct loc_network_list* list) { -+ struct loc_network_list* to_check; -+ -+ // Create a new list with all networks to look at -+ int r = loc_network_list_new(network->ctx, &to_check); -+ if (r) -+ return NULL; -+ -+ struct loc_network* subnet = NULL; -+ struct loc_network_list* subnets = NULL; -+ -+ for (unsigned int i = 0; i < loc_network_list_size(list); i++) { -+ subnet = loc_network_list_get(list, i); -+ -+ // Find all excluded networks -+ struct loc_network_list* excluded = loc_network_exclude(network, subnet); -+ if (excluded) { -+ // Add them all to the "to check" list -+ loc_network_list_merge(to_check, excluded); -+ loc_network_list_unref(excluded); -+ } -+ -+ // Cleanup -+ loc_network_unref(subnet); -+ } -+ -+ r = loc_network_list_new(network->ctx, &subnets); -+ if (r) { -+ loc_network_list_unref(to_check); -+ return NULL; -+ } -+ -+ while (!loc_network_list_empty(to_check)) { -+ struct loc_network* subnet_to_check = loc_network_list_pop(to_check); -+ -+ // Marks whether this subnet passed all checks -+ int passed = 1; -+ -+ for (unsigned int i = 0; i < loc_network_list_size(list); i++) { -+ subnet = loc_network_list_get(list, i); -+ -+ // Drop this subnet if is is already in list -+ if (loc_network_eq(subnet_to_check, subnet)) { -+ passed = 0; -+ loc_network_unref(subnet); -+ break; -+ } -+ -+ // Drop this subnet if is a subnet of another subnet -+ if (loc_network_is_subnet_of(subnet, subnet_to_check)) { -+ passed = 0; -+ loc_network_unref(subnet); -+ break; -+ } -+ -+ // Break it down if it overlaps -+ if (loc_network_overlaps(subnet_to_check, subnet)) { -+ passed = 0; -+ -+ struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet); -+ if (excluded) { -+ loc_network_list_merge(to_check, excluded); -+ loc_network_list_unref(excluded); -+ } -+ -+ loc_network_unref(subnet); -+ break; -+ } -+ -+ loc_network_unref(subnet); -+ } -+ -+ if (passed) { -+ r = loc_network_list_push(subnets, subnet_to_check); -+ } -+ -+ loc_network_unref(subnet_to_check); -+ } -+ -+ loc_network_list_unref(to_check); -+ -+ return subnets; -+} -+ - LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { - // Add country code - loc_country_code_copy(dbobj->country_code, network->country_code); --- -2.20.1 - -From d87fd7a3d277b4b03222c7d1680e51b3e45e525b Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 20:02:03 +0000 -Subject: [PATCH 046/111] database: Add option to return networks flattened - -Signed-off-by: Michael Tremer ---- - src/database.c | 174 ++++++++++++++++++++++++++++++++++++++++--------- - 1 file changed, 143 insertions(+), 31 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 9baab33..7a3d1a7 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -118,6 +118,9 @@ struct loc_database_enumerator { - struct loc_node_stack network_stack[MAX_STACK_DEPTH]; - int network_stack_depth; - unsigned int* networks_visited; -+ -+ // For subnet search -+ struct loc_network_list* stack; - }; - - static int loc_database_read_magic(struct loc_database* db) { -@@ -935,6 +938,26 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, - - // Enumerator - -+static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { -+ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); -+ -+ // Release all references -+ loc_database_unref(enumerator->db); -+ loc_unref(enumerator->ctx); +diff --git a/src/country-list.c b/src/country-list.c +new file mode 100644 +index 0000000..cc36740 +--- /dev/null ++++ b/src/country-list.c +@@ -0,0 +1,161 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet + -+ if (enumerator->string) -+ free(enumerator->string); ++ Copyright (C) 2020 IPFire Development Team + -+ // Free network search -+ free(enumerator->networks_visited); ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. + -+ // Free subnet stack -+ if (enumerator->stack) -+ loc_network_list_unref(enumerator->stack); ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ + -+ free(enumerator); -+} ++#include ++#include + - LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, - struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { - struct loc_database_enumerator* e = calloc(1, sizeof(*e)); -@@ -951,10 +974,16 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum - e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); - - // Initialise graph search -- //e->network_stack[++e->network_stack_depth] = 0; - e->network_stack_depth = 1; - e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited)); - -+ // Allocate stack -+ int r = loc_network_list_new(e->ctx, &e->stack); -+ if (r) { -+ loc_database_enumerator_free(e); -+ return r; -+ } ++#include ++#include ++#include + - DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e); - - *enumerator = e; -@@ -967,22 +996,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo - return enumerator; - } - --static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { -- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); -- -- // Release all references -- loc_database_unref(enumerator->db); -- loc_unref(enumerator->ctx); -- -- if (enumerator->string) -- free(enumerator->string); -- -- // Free network search -- free(enumerator->networks_visited); -- -- free(enumerator); --} -- - LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) { - if (!enumerator) - return NULL; -@@ -1116,17 +1129,13 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } - --LOC_EXPORT int loc_database_enumerator_next_network( -- struct loc_database_enumerator* enumerator, struct loc_network** network) { -- // Reset network -- *network = NULL; -- -- // Do not do anything if not in network mode -- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) -+static int __loc_database_enumerator_next_network( -+ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { -+ // Return top element from the stack -+ *network = loc_network_list_pop(enumerator->stack); -+ if (*network) - return 0; - -- int r; -- - DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", - enumerator->network_stack_depth); - -@@ -1155,7 +1164,7 @@ LOC_EXPORT int loc_database_enumerator_next_network( - enumerator->db->network_nodes_v1 + node->offset; - - // Add edges to stack -- r = loc_database_enumerator_stack_push_node(enumerator, -+ int r = loc_database_enumerator_stack_push_node(enumerator, - be32toh(n->one), 1, node->depth + 1); - - if (r) -@@ -1181,6 +1190,10 @@ LOC_EXPORT int loc_database_enumerator_next_network( - if (r) - return r; - -+ // Return all networks when the filter is disabled -+ if (!filter) -+ return 0; ++struct loc_country_list { ++ struct loc_ctx* ctx; ++ int refcount; + - // Check if we are interested in this network - - // Skip if the family does not match -@@ -1223,12 +1236,111 @@ LOC_EXPORT int loc_database_enumerator_next_network( - } - - // Reached the end of the search -+ return 0; -+} - -- // Mark all nodes as non-visited -- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++) -- enumerator->networks_visited[i] = 0; -+static int __loc_database_enumerator_next_network_flattened( -+ struct loc_database_enumerator* enumerator, struct loc_network** network) { -+ // Fetch the next network -+ int r = __loc_database_enumerator_next_network(enumerator, network, 1); -+ if (r) -+ return r; - -- return 0; -+ // End if we could not read another network -+ if (!*network) -+ return 0; ++ struct loc_country** elements; ++ size_t elements_size; + -+ struct loc_network* subnet = NULL; -+ struct loc_network_list* subnets; ++ size_t size; ++}; + -+ // Create a list with all subnets -+ r = loc_network_list_new(enumerator->ctx, &subnets); -+ if (r) -+ return r; ++static int loc_country_list_grow(struct loc_country_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); + -+ // Search all subnets from the database -+ while (1) { -+ // Fetch the next network in line -+ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0); -+ if (r) -+ goto END; ++ struct loc_country** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; + -+ // End if we did not receive another subnet -+ if (!subnet) -+ break; ++ list->elements = elements; ++ list->elements_size += size; + -+ // Collect all subnets in a list -+ if (loc_network_is_subnet(*network, subnet)) { -+ r = loc_network_list_push(subnets, subnet); -+ if (r) -+ goto END; ++ return 0; ++} + -+ loc_network_unref(subnet); -+ continue; -+ } ++LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, ++ struct loc_country_list** list) { ++ struct loc_country_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; + -+ // If this is not a subnet, we push it back onto the stack and break -+ r = loc_network_list_push(enumerator->stack, subnet); -+ if (r) -+ goto END; ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; + -+ loc_network_unref(subnet); -+ break; -+ } ++ DEBUG(l->ctx, "Country list allocated at %p\n", l); ++ *list = l; + -+ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets)); ++ return 0; ++} + -+ // We can abort here if the network has no subnets -+ if (loc_network_list_empty(subnets)) { -+ loc_network_list_unref(subnets); ++LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) { ++ list->refcount++; + -+ return 0; -+ } ++ return list; ++} + -+ // If the network has any subnets, we will break it into smaller parts -+ // without the subnets. -+ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets); -+ if (!excluded || loc_network_list_empty(excluded)) { -+ r = 1; -+ goto END; -+ } ++static void loc_country_list_free(struct loc_country_list* list) { ++ DEBUG(list->ctx, "Releasing country list at %p\n", list); + -+ // Sort the result -+ loc_network_list_sort(excluded); ++ loc_country_list_clear(list); + -+ // Reverse the list -+ loc_network_list_reverse(excluded); ++ loc_unref(list->ctx); ++ free(list); ++} + -+ // Replace network with the first one -+ loc_network_unref(*network); ++LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) { ++ if (!list) ++ return NULL; + -+ *network = loc_network_list_pop(excluded); ++ if (--list->refcount > 0) ++ return list; + -+ // Push the rest onto the stack -+ loc_network_list_merge(enumerator->stack, excluded); ++ loc_country_list_free(list); ++ return NULL; ++} + -+ loc_network_list_unref(excluded); ++LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) { ++ return list->size; ++} + -+END: -+ if (subnet) -+ loc_network_unref(subnet); ++LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { ++ return list->size == 0; ++} + -+ loc_network_list_unref(subnets); ++LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { ++ if (!list->elements) ++ return; + -+ return r; ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_country_unref(list->elements[i]); ++ ++ free(list->elements); ++ list->elements = NULL; ++ list->elements_size = 0; ++ ++ list->size = 0; +} + -+LOC_EXPORT int loc_database_enumerator_next_network( -+ struct loc_database_enumerator* enumerator, struct loc_network** network) { -+ // Do not do anything if not in network mode -+ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) -+ return 0; ++LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; + -+ // Flatten output? -+ if (enumerator->flatten) -+ return __loc_database_enumerator_next_network_flattened(enumerator, network); ++ return loc_country_ref(list->elements[index]); ++} + -+ return __loc_database_enumerator_next_network(enumerator, network, 1); - } - - LOC_EXPORT int loc_database_enumerator_next_country( --- -2.20.1 - -From d3ae93c27dcd7f6984fdc29cc141621e277f2e2a Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 20:09:20 +0000 -Subject: [PATCH 047/111] test: Update API - -Signed-off-by: Michael Tremer ---- - src/test-as.c | 2 +- - src/test-database.c | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/test-as.c b/src/test-as.c -index 839a04c..2d61675 100644 ---- a/src/test-as.c -+++ b/src/test-as.c -@@ -95,7 +95,7 @@ int main(int argc, char** argv) { - // Enumerator - - struct loc_database_enumerator* enumerator; -- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES); -+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0); - if (err) { - fprintf(stderr, "Could not create a database enumerator\n"); - exit(EXIT_FAILURE); -diff --git a/src/test-database.c b/src/test-database.c -index 4aef94e..da4b11c 100644 ---- a/src/test-database.c -+++ b/src/test-database.c -@@ -198,7 +198,7 @@ int main(int argc, char** argv) { - - // Enumerator - struct loc_database_enumerator* enumerator; -- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS); -+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0); - if (err) { - fprintf(stderr, "Could not initialise the enumerator: %d\n", err); - exit(EXIT_FAILURE); --- -2.20.1 - -From 594ca328c6e124d0f1eb543e9c8d9bbfe8a7b628 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 12 Nov 2020 20:09:37 +0000 -Subject: [PATCH 048/111] networks: Copy all attributes when splitting networks - -Signed-off-by: Michael Tremer ---- - src/network.c | 14 ++++++++++++++ - 1 file changed, 14 insertions(+) - -diff --git a/src/network.c b/src/network.c -index 751e8e5..d67f116 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -550,6 +550,20 @@ LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* netw - if (r) - goto ERROR; - -+ // Copy country code -+ const char* country_code = loc_network_get_country_code(network); -+ if (country_code) { -+ loc_network_set_country_code(subnet1, country_code); -+ loc_network_set_country_code(subnet2, country_code); -+ } ++LOC_EXPORT int loc_country_list_append( ++ struct loc_country_list* list, struct loc_country* country) { ++ if (loc_country_list_contains(list, country)) ++ return 0; + -+ // Copy ASN -+ uint32_t asn = loc_network_get_asn(network); -+ if (asn) { -+ loc_network_set_asn(subnet1, asn); -+ loc_network_set_asn(subnet2, asn); ++ // Check if we have space left ++ if (list->size >= list->elements_size) { ++ int r = loc_country_list_grow(list, 64); ++ if (r) ++ return r; + } + - loc_network_unref(subnet1); - loc_network_unref(subnet2); - --- -2.20.1 - -From 69248038292e9ea1a4ee8912cdfc8700456753ad Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 13 Nov 2020 11:23:33 +0000 -Subject: [PATCH 049/111] database: Move network filtering into a separate - function - -Signed-off-by: Michael Tremer ---- - src/database.c | 56 +++++++++++++++++++++++--------------------------- - 1 file changed, 26 insertions(+), 30 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 7a3d1a7..72bc8eb 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1129,6 +1129,31 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } - -+static int loc_database_enumerator_filter_network( -+ struct loc_database_enumerator* enumerator, struct loc_network* network) { -+ // Skip if the family does not match -+ if (enumerator->family && loc_network_address_family(network) != enumerator->family) -+ return 1; ++ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); + -+ // Skip if the country code does not match -+ if (*enumerator->country_code && -+ !loc_network_match_country_code(network, enumerator->country_code)) -+ return 1; ++ list->elements[list->size++] = loc_country_ref(country); + -+ // Skip if the ASN does not match -+ if (enumerator->asn && -+ !loc_network_match_asn(network, enumerator->asn)) -+ return 1; ++ return 0; ++} + -+ // Skip if flags do not match -+ if (enumerator->flags && -+ !loc_network_match_flag(network, enumerator->flags)) -+ return 1; ++LOC_EXPORT int loc_country_list_contains( ++ struct loc_country_list* list, struct loc_country* country) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_country_cmp(country, list->elements[i]) == 0) ++ return 1; ++ } + -+ // Do not filter + return 0; +} + - static int __loc_database_enumerator_next_network( - struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { - // Return top element from the stack -@@ -1195,36 +1220,7 @@ static int __loc_database_enumerator_next_network( - return 0; - - // Check if we are interested in this network -- -- // Skip if the family does not match -- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) { -- loc_network_unref(*network); -- *network = NULL; -- -- continue; -- } -- -- // Skip if the country code does not match -- if (*enumerator->country_code && -- !loc_network_match_country_code(*network, enumerator->country_code)) { -- loc_network_unref(*network); -- *network = NULL; -- -- continue; -- } -- -- // Skip if the ASN does not match -- if (enumerator->asn && -- !loc_network_match_asn(*network, enumerator->asn)) { -- loc_network_unref(*network); -- *network = NULL; -- -- continue; -- } -- -- // Skip if flags do not match -- if (enumerator->flags && -- !loc_network_match_flag(*network, enumerator->flags)) { -+ if (loc_database_enumerator_filter_network(enumerator, *network)) { - loc_network_unref(*network); - *network = NULL; - --- -2.20.1 - -From 2113e71bf7b997c82670c5c22cf91aa6442fe6f3 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 13 Nov 2020 11:29:02 +0000 -Subject: [PATCH 050/111] database: Filter results coming from stack - -Signed-off-by: Michael Tremer ---- - src/database.c | 18 ++++++++++++++++-- - 1 file changed, 16 insertions(+), 2 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 72bc8eb..0f3cdc2 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1157,9 +1157,23 @@ static int loc_database_enumerator_filter_network( - static int __loc_database_enumerator_next_network( - struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { - // Return top element from the stack -- *network = loc_network_list_pop(enumerator->stack); -- if (*network) -+ while (1) { -+ *network = loc_network_list_pop(enumerator->stack); ++LOC_EXPORT int loc_country_list_contains_code( ++ struct loc_country_list* list, const char* code) { ++ struct loc_country* country; + -+ // Stack is empty -+ if (!*network) -+ break; ++ int r = loc_country_new(list->ctx, &country, code); ++ if (r) ++ return -1; + -+ // Throw away any networks by filter -+ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) { -+ loc_network_unref(*network); -+ *network = NULL; -+ continue; -+ } ++ r = loc_country_list_contains(list, country); ++ loc_country_unref(country); + -+ // Return result - return 0; -+ } ++ return r; ++} +diff --git a/src/country.c b/src/country.c +index 2ba93e6..7aac0db 100644 +--- a/src/country.c ++++ b/src/country.c +@@ -34,6 +34,9 @@ struct loc_country { + }; - DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", - enumerator->network_stack_depth); --- -2.20.1 - -From d33753688138c9938743dafbbdddf220dd2afd14 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 13 Nov 2020 11:29:15 +0000 -Subject: [PATCH 051/111] network: Sort result of excluded lists - -Signed-off-by: Michael Tremer ---- - src/database.c | 3 --- - src/network.c | 3 +++ - 2 files changed, 3 insertions(+), 3 deletions(-) - + LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) { ++ if (!loc_country_code_is_valid(country_code)) ++ return -EINVAL; ++ + struct loc_country* c = calloc(1, sizeof(*c)); + if (!c) + return -ENOMEM; diff --git a/src/database.c b/src/database.c -index 0f3cdc2..6849d97 100644 +index fa1dad0..1871b74 100644 --- a/src/database.c +++ b/src/database.c -@@ -1315,9 +1315,6 @@ static int __loc_database_enumerator_next_network_flattened( - goto END; - } +@@ -38,8 +38,10 @@ -- // Sort the result -- loc_network_list_sort(excluded); -- - // Reverse the list - loc_network_list_reverse(excluded); + #include + #include ++#include + #include + #include ++#include + #include + #include + #include +@@ -99,11 +101,14 @@ struct loc_database_enumerator { -diff --git a/src/network.c b/src/network.c -index d67f116..9d02bf8 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -773,6 +773,9 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( + // Search string + char* string; +- char country_code[3]; +- uint32_t asn; ++ struct loc_country_list* countries; ++ struct loc_as_list* asns; + enum loc_network_flags flags; + int family; - loc_network_list_unref(to_check); ++ // Flatten output? ++ int flatten; ++ + // Index of the AS we are looking at + unsigned int as_index; -+ // Sort the result -+ loc_network_list_sort(subnets); +@@ -115,6 +120,9 @@ struct loc_database_enumerator { + struct loc_node_stack network_stack[MAX_STACK_DEPTH]; + int network_stack_depth; + unsigned int* networks_visited; + - return subnets; - } ++ // For subnet search ++ struct loc_network_list* stack; + }; --- -2.20.1 - -From 8d777f12f7ffa4df1b28d197563888296803b727 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 13 Nov 2020 11:38:15 +0000 -Subject: [PATCH 052/111] network: Add function to pop first element from stack - -Signed-off-by: Michael Tremer ---- - src/database.c | 6 ++---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 19 +++++++++++++++++++ - 4 files changed, 23 insertions(+), 4 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 6849d97..b9d870f 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1315,15 +1315,13 @@ static int __loc_database_enumerator_next_network_flattened( - goto END; + static int loc_database_read_magic(struct loc_database* db) { +@@ -611,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { } -- // Reverse the list -- loc_network_list_reverse(excluded); -- - // Replace network with the first one - loc_network_unref(*network); + clock_t end = clock(); +- DEBUG(db->ctx, "Signature checked in %.4fms\n", ++ INFO(db->ctx, "Signature checked in %.4fms\n", + (double)(end - start) / CLOCKS_PER_SEC * 1000); -- *network = loc_network_list_pop(excluded); -+ *network = loc_network_list_pop_first(excluded); + CLEANUP: +@@ -671,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, + off_t lo = 0; + off_t hi = db->as_count - 1; - // Push the rest onto the stack -+ loc_network_list_reverse(excluded); - loc_network_list_merge(enumerator->stack, excluded); ++#ifdef ENABLE_DEBUG + // Save start time + clock_t start = clock(); ++#endif - loc_network_list_unref(excluded); -diff --git a/src/libloc.sym b/src/libloc.sym -index bcd11be..6139db6 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -113,6 +113,7 @@ global: - loc_network_list_merge; - loc_network_list_new; - loc_network_list_pop; -+ loc_network_list_pop_first; - loc_network_list_push; - loc_network_list_ref; - loc_network_list_reverse; -diff --git a/src/loc/network.h b/src/loc/network.h -index 40712b9..203e61c 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -76,6 +76,7 @@ void loc_network_list_dump(struct loc_network_list* list); - struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); - int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); - struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); - int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); - void loc_network_list_sort(struct loc_network_list* list); - void loc_network_list_reverse(struct loc_network_list* list); -diff --git a/src/network.c b/src/network.c -index 9d02bf8..e7dc97e 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -1231,6 +1231,25 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis - return network; - } + while (lo <= hi) { + off_t i = (lo + hi) / 2; +@@ -685,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, + // Check if this is a match + uint32_t as_number = loc_as_get_number(*as); + if (as_number == number) { ++#ifdef ENABLE_DEBUG + clock_t end = clock(); -+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); -+ return NULL; -+ } -+ -+ struct loc_network* network = list->list[0]; -+ -+ // Move all elements to the top of the stack -+ for (unsigned int i = 0; i < --list->size; i++) { -+ list->list[i] = list->list[i+1]; -+ } -+ -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -+ -+ return network; -+} -+ - LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { - for (unsigned int i = 0; i < list->size; i++) { - if (loc_network_eq(list->list[i], network)) --- -2.20.1 - -From 7933f5bfb4dd7603cb646e192840762bf6394292 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 13 Nov 2020 11:43:53 +0000 -Subject: [PATCH 053/111] network: Unexport all tree functions - -These should not be exported - -Signed-off-by: Michael Tremer ---- - src/network.c | 28 ++++++++++++++-------------- - src/test-network.c | 9 +++++++++ - 2 files changed, 23 insertions(+), 14 deletions(-) - -diff --git a/src/network.c b/src/network.c -index e7dc97e..d015579 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -847,7 +847,7 @@ struct loc_network_tree_node { - struct loc_network* network; - }; + // Log how fast this has been + DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number, + (double)(end - start) / CLOCKS_PER_SEC * 1000); ++#endif --LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { -+int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { - struct loc_network_tree* t = calloc(1, sizeof(*t)); - if (!t) - return -ENOMEM; -@@ -867,7 +867,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree - return 0; - } + return 0; + } +@@ -733,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ + return -1; + } --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { -+struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { - return loc_network_tree_node_ref(tree->root); - } ++#ifdef ENABLE_DEBUG + if (r == 0) { + char* string = loc_network_str(*network); + DEBUG(db->ctx, "Got network %s\n", string); + free(string); + } ++#endif -@@ -939,7 +939,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_ - return 0; + return r; } +@@ -762,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru + } --LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree, -+int loc_network_tree_walk(struct loc_network_tree* tree, - int(*filter_callback)(struct loc_network* network, void* data), - int(*callback)(struct loc_network* network, void* data), void* data) { - return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data); -@@ -954,7 +954,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) { - free(tree); - } + // Check if the given IP address is inside the network +- r = loc_network_match_address(*network, address); +- if (r) { ++ if (!loc_network_match_address(*network, address)) { + DEBUG(db->ctx, "Searched address is not part of the network\n"); --LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { -+struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { - if (--tree->refcount > 0) - return tree; + loc_network_unref(*network); +@@ -832,17 +845,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db, -@@ -975,13 +975,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) { - return 0; - } + *network = NULL; --LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) { -+int loc_network_tree_dump(struct loc_network_tree* tree) { - DEBUG(tree->ctx, "Dumping network tree at %p\n", tree); ++#ifdef ENABLE_DEBUG + // Save start time + clock_t start = clock(); ++#endif - return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL); - } + int r = __loc_database_lookup(db, address, network, &network_address, + db->network_nodes_v1, 0); --LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { -+int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { - DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree); ++#ifdef ENABLE_DEBUG + clock_t end = clock(); - struct loc_network_tree_node* node = loc_network_tree_get_path(tree, -@@ -1012,7 +1012,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) { - return 0; + // Log how fast this has been + DEBUG(db->ctx, "Executed network search in %.4fms\n", + (double)(end - start) / CLOCKS_PER_SEC * 1000); ++#endif + + return r; } +@@ -889,8 +906,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, + off_t lo = 0; + off_t hi = db->countries_count - 1; --LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { -+size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { - size_t counter = 0; ++#ifdef ENABLE_DEBUG + // Save start time + clock_t start = clock(); ++#endif + + while (lo <= hi) { + off_t i = (lo + hi) / 2; +@@ -905,11 +924,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, + int result = strcmp(code, cc); + + if (result == 0) { ++#ifdef ENABLE_DEBUG + clock_t end = clock(); + + // Log how fast this has been + DEBUG(db->ctx, "Found country %s in %.4fms\n", cc, + (double)(end - start) / CLOCKS_PER_SEC * 1000); ++#endif + + return 0; + } +@@ -932,8 +953,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, + + // Enumerator + ++static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { ++ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); ++ ++ // Release all references ++ loc_database_unref(enumerator->db); ++ loc_unref(enumerator->ctx); ++ ++ if (enumerator->string) ++ free(enumerator->string); ++ ++ if (enumerator->countries) ++ loc_country_list_unref(enumerator->countries); ++ ++ if (enumerator->asns) ++ loc_as_list_unref(enumerator->asns); ++ ++ // Free network search ++ free(enumerator->networks_visited); ++ ++ // Free subnet stack ++ if (enumerator->stack) ++ loc_network_list_unref(enumerator->stack); ++ ++ free(enumerator); ++} ++ + LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, +- struct loc_database* db, enum loc_database_enumerator_mode mode) { ++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { + struct loc_database_enumerator* e = calloc(1, sizeof(*e)); + if (!e) + return -ENOMEM; +@@ -944,11 +991,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum + e->mode = mode; + e->refcount = 1; + ++ // Flatten output? ++ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); ++ + // Initialise graph search +- //e->network_stack[++e->network_stack_depth] = 0; + e->network_stack_depth = 1; + e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited)); - int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter); -@@ -1034,11 +1034,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node) - return counter; - } ++ // Allocate stack ++ int r = loc_network_list_new(e->ctx, &e->stack); ++ if (r) { ++ loc_database_enumerator_free(e); ++ return r; ++ } ++ + DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e); --LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { -+size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { - return __loc_network_tree_count_nodes(tree->root); + *enumerator = e; +@@ -961,22 +1017,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo + return enumerator; } --LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { -+int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { - struct loc_network_tree_node* n = calloc(1, sizeof(*n)); - if (!n) - return -ENOMEM; -@@ -1053,7 +1053,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network +-static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { +- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); +- +- // Release all references +- loc_database_unref(enumerator->db); +- loc_unref(enumerator->ctx); +- +- if (enumerator->string) +- free(enumerator->string); +- +- // Free network search +- free(enumerator->networks_visited); +- +- free(enumerator); +-} +- + LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) { + if (!enumerator) + return NULL; +@@ -998,40 +1038,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator return 0; } --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { -+struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { - if (node) - node->refcount++; +-LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) { +- // Set empty country code +- if (!country_code || !*country_code) { +- *enumerator->country_code = '\0'; +- return 0; +- } ++LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries( ++ struct loc_database_enumerator* enumerator) { ++ if (!enumerator->countries) ++ return NULL; -@@ -1076,7 +1076,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) { - free(node); - } +- // Treat A1, A2, A3 as special country codes, +- // but perform search for flags instead +- if (strcmp(country_code, "A1") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_ANONYMOUS_PROXY); +- } else if (strcmp(country_code, "A2") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_SATELLITE_PROVIDER); +- } else if (strcmp(country_code, "A3") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_ANYCAST); +- } ++ return loc_country_list_ref(enumerator->countries); ++} --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { -+struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { - if (!node) - return NULL; +- // Country codes must be two characters +- if (!loc_country_code_is_valid(country_code)) +- return -EINVAL; ++LOC_EXPORT int loc_database_enumerator_set_countries( ++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) { ++ if (enumerator->countries) ++ loc_country_list_unref(enumerator->countries); -@@ -1087,7 +1087,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_ - return NULL; - } +- for (unsigned int i = 0; i < 3; i++) { +- enumerator->country_code[i] = country_code[i]; +- } ++ enumerator->countries = loc_country_list_ref(countries); --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { -+struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { - if (index == 0) - node = node->zero; - else -@@ -1099,11 +1099,11 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne - return loc_network_tree_node_ref(node); + return 0; } --LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { -+int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { - return (!!node->network); - } +-LOC_EXPORT int loc_database_enumerator_set_asn( +- struct loc_database_enumerator* enumerator, unsigned int asn) { +- enumerator->asn = asn; ++LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns( ++ struct loc_database_enumerator* enumerator) { ++ if (!enumerator->asns) ++ return NULL; ++ ++ return loc_as_list_ref(enumerator->asns); ++} ++ ++LOC_EXPORT int loc_database_enumerator_set_asns( ++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) { ++ if (enumerator->asns) ++ loc_as_list_unref(enumerator->asns); ++ ++ enumerator->asns = loc_as_list_ref(asns); --LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { -+struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { - return loc_network_ref(node->network); + return 0; + } +@@ -1110,16 +1148,64 @@ static int loc_database_enumerator_stack_push_node( + return 0; } -diff --git a/src/test-network.c b/src/test-network.c -index af1b2e6..7c90224 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -37,12 +37,14 @@ int main(int argc, char** argv) { - // Enable debug logging - loc_set_log_priority(ctx, LOG_DEBUG); - -+#if 0 - struct loc_network_tree* tree; - err = loc_network_tree_new(ctx, &tree); - if (err) { - fprintf(stderr, "Could not create the network tree\n"); - exit(EXIT_FAILURE); - } -+#endif +-LOC_EXPORT int loc_database_enumerator_next_network( +- struct loc_database_enumerator* enumerator, struct loc_network** network) { +- // Reset network +- *network = NULL; ++static int loc_database_enumerator_filter_network( ++ struct loc_database_enumerator* enumerator, struct loc_network* network) { ++ // Skip if the family does not match ++ if (enumerator->family && loc_network_address_family(network) != enumerator->family) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network); ++ return 1; ++ } - // Create a network - struct loc_network* network1; -@@ -58,12 +60,14 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } +- // Do not do anything if not in network mode +- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) +- return 0; ++ // Skip if the country code does not match ++ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) { ++ const char* country_code = loc_network_get_country_code(network); -+#if 0 - // Adding network to the tree - err = loc_network_tree_add_network(tree, network1); - if (err) { - fprintf(stderr, "Could not add network to the tree\n"); - exit(EXIT_FAILURE); - } -+#endif +- int r; ++ if (!loc_country_list_contains_code(enumerator->countries, country_code)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network); ++ return 1; ++ } ++ } ++ ++ // Skip if the ASN does not match ++ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) { ++ uint32_t asn = loc_network_get_asn(network); ++ ++ if (!loc_as_list_contains_number(enumerator->asns, asn)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network); ++ return 1; ++ } ++ } ++ ++ // Skip if flags do not match ++ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network); ++ return 1; ++ } ++ ++ // Do not filter ++ return 0; ++} ++ ++static int __loc_database_enumerator_next_network( ++ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { ++ // Return top element from the stack ++ while (1) { ++ *network = loc_network_list_pop_first(enumerator->stack); ++ ++ // Stack is empty ++ if (!*network) ++ break; ++ ++ // Throw away any networks by filter ++ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) { ++ loc_network_unref(*network); ++ *network = NULL; ++ continue; ++ } ++ ++ // Return result ++ return 0; ++ } - // Check if the first and last addresses are correct - char* string = loc_network_format_first_address(network1); -@@ -101,6 +105,7 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } + DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", + enumerator->network_stack_depth); +@@ -1149,7 +1235,7 @@ LOC_EXPORT int loc_database_enumerator_next_network( + enumerator->db->network_nodes_v1 + node->offset; -+#if 0 - // Adding network to the tree - err = loc_network_tree_add_network(tree, network2); - if (err) { -@@ -117,6 +122,7 @@ int main(int argc, char** argv) { + // Add edges to stack +- r = loc_database_enumerator_stack_push_node(enumerator, ++ int r = loc_database_enumerator_stack_push_node(enumerator, + be32toh(n->one), 1, node->depth + 1); - size_t nodes = loc_network_tree_count_nodes(tree); - printf("The tree has %zu nodes\n", nodes); -+#endif + if (r) +@@ -1175,56 +1261,145 @@ LOC_EXPORT int loc_database_enumerator_next_network( + if (r) + return r; - // Check equals function - err = loc_network_eq(network1, network1); -@@ -260,7 +266,10 @@ int main(int argc, char** argv) { - loc_network_unref(network2); - loc_network_unref(network3); - loc_network_unref(network4); -+ -+#if 0 - loc_network_tree_unref(tree); -+#endif +- // Check if we are interested in this network ++ // Return all networks when the filter is disabled ++ if (!filter) ++ return 0; - // And open it again from disk - struct loc_database* db; --- -2.20.1 - -From c242f7325bd6fc4ba26047ac24196d1c161c6e01 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 13 Nov 2020 12:09:03 +0000 -Subject: [PATCH 054/111] python: Move tree flattening into C - -Signed-off-by: Michael Tremer ---- - src/python/database.c | 8 ++- - src/python/export.py | 138 +++++------------------------------------- - 2 files changed, 21 insertions(+), 125 deletions(-) - -diff --git a/src/python/database.c b/src/python/database.c -index 7f8c2c2..d169547 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -258,17 +258,19 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { - } +- // Skip if the family does not match +- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) { ++ // Check if we are interested in this network ++ if (loc_database_enumerator_filter_network(enumerator, *network)) { + loc_network_unref(*network); + *network = NULL; - static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { -- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL }; -+ char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL }; - const char* country_code = NULL; - unsigned int asn = 0; - int flags = 0; - int family = 0; -+ int flatten = 0; + continue; + } -- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family)) -+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten)) - return NULL; +- // Skip if the country code does not match +- if (*enumerator->country_code && +- !loc_network_match_country_code(*network, enumerator->country_code)) { +- loc_network_unref(*network); +- *network = NULL; ++ return 0; ++ } ++ } - struct loc_database_enumerator* enumerator; -- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0); -+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, -+ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -diff --git a/src/python/export.py b/src/python/export.py -index dd44332..be4a68e 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -87,58 +87,12 @@ class OutputWriter(object): - def _write_network(self, network): - self.f.write("%s\n" % network) +- continue; +- } ++ // Reached the end of the search ++ return 0; ++} -- def write(self, network, subnets): -+ def write(self, network): - if self.flatten and self._flatten(network): - log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network)) - return +- // Skip if the ASN does not match +- if (enumerator->asn && +- !loc_network_match_asn(*network, enumerator->asn)) { +- loc_network_unref(*network); +- *network = NULL; ++static int __loc_database_enumerator_next_network_flattened( ++ struct loc_database_enumerator* enumerator, struct loc_network** network) { ++ // Fetch the next network ++ int r = __loc_database_enumerator_next_network(enumerator, network, 1); ++ if (r) ++ return r; -- # Convert network into a Python object -- _network = ipaddress.ip_network("%s" % network) -- -- # Write the network when it has no subnets -- if not subnets: -- log.debug("Writing %s to %s" % (_network, self.f)) -- return self._write_network(_network) -- -- # Convert subnets into Python objects -- _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets] -- -- # Split the network into smaller bits so that -- # we can accomodate for any gaps in it later -- to_check = set() -- for _subnet in _subnets: -- to_check.update( -- _network.address_exclude(_subnet) -- ) -- -- # Clear the list of all subnets -- subnets = [] -- -- # Check if all subnets to not overlap with anything else -- while to_check: -- subnet_to_check = to_check.pop() -- -- for _subnet in _subnets: -- # Drop this subnet if it equals one of the subnets -- # or if it is subnet of one of them -- if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet): -- break -- -- # Break it down if it overlaps -- if subnet_to_check.overlaps(_subnet): -- to_check.update( -- subnet_to_check.address_exclude(_subnet) -- ) -- break -- -- # Add the subnet again as it passed the check -- else: -- subnets.append(subnet_to_check) -- -- # Write all networks as compact as possible -- for network in ipaddress.collapse_addresses(subnets): -- log.debug("Writing %s to %s" % (network, self.f)) -- self._write_network(network) -+ return self._write_network(network) +- continue; +- } ++ // End if we could not read another network ++ if (!*network) ++ return 0; - def finish(self): - """ -@@ -188,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter): - mode = "wb" +- // Skip if flags do not match +- if (enumerator->flags && +- !loc_network_match_flag(*network, enumerator->flags)) { +- loc_network_unref(*network); +- *network = NULL; ++ struct loc_network* subnet = NULL; ++ struct loc_network_list* subnets; - def _write_network(self, network): -- for address in (network.network_address, network.broadcast_address): -+ for address in (network.first_address, network.last_address): - # Convert this into a string of bits - bytes = socket.inet_pton( - socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, -@@ -231,42 +185,21 @@ class Exporter(object): - writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) +- continue; ++ // Create a list with all subnets ++ r = loc_network_list_new(enumerator->ctx, &subnets); ++ if (r) ++ return r; ++ ++ // Search all subnets from the database ++ while (1) { ++ // Fetch the next network in line ++ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0); ++ if (r) { ++ loc_network_unref(subnet); ++ loc_network_list_unref(subnets); ++ ++ return r; ++ } ++ ++ // End if we did not receive another subnet ++ if (!subnet) ++ break; ++ ++ // Collect all subnets in a list ++ if (loc_network_is_subnet(*network, subnet)) { ++ r = loc_network_list_push(subnets, subnet); ++ if (r) { ++ loc_network_unref(subnet); ++ loc_network_list_unref(subnets); ++ ++ return r; + } - # Get all networks that match the family -- networks = self.db.search_networks(family=family) -- -- # Create a stack with all networks in order where we can put items back -- # again and retrieve them in the next iteration. -- networks = BufferedStack(networks) -+ networks = self.db.search_networks(family=family, flatten=True) +- return 0; ++ loc_network_unref(subnet); ++ continue; + } ++ ++ // If this is not a subnet, we push it back onto the stack and break ++ r = loc_network_list_push(enumerator->stack, subnet); ++ if (r) { ++ loc_network_unref(subnet); ++ loc_network_list_unref(subnets); ++ ++ return r; ++ } ++ ++ loc_network_unref(subnet); ++ break; + } - # Walk through all networks - for network in networks: -- # Collect all networks which are a subnet of network -- subnets = [] -- for subnet in networks: -- # If the next subnet was not a subnet, we have to push -- # it back on the stack and break this loop -- if not subnet.is_subnet_of(network): -- networks.push(subnet) -- break -- -- subnets.append(subnet) -- - # Write matching countries -- if network.country_code and network.country_code in writers: -- # Mismatching subnets -- gaps = [ -- subnet for subnet in subnets if not network.country_code == subnet.country_code -- ] -- -- writers[network.country_code].write(network, gaps) -+ try: -+ writers[network.country_code].write(network) -+ except KeyError: -+ pass - - # Write matching ASNs -- if network.asn and network.asn in writers: -- # Mismatching subnets -- gaps = [ -- subnet for subnet in subnets if not network.asn == subnet.asn -- ] -- -- writers[network.asn].write(network, gaps) -+ try: -+ writers[network.asn].write(network) -+ except KeyError: -+ pass +- // Reached the end of the search ++ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets)); ++ ++ // We can abort here if the network has no subnets ++ if (loc_network_list_empty(subnets)) { ++ loc_network_list_unref(subnets); ++ ++ return 0; ++ } - # Handle flags - for flag in flags: -@@ -274,19 +207,10 @@ class Exporter(object): - # Fetch the "fake" country code - country = flags[flag] +- // Mark all nodes as non-visited +- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++) +- enumerator->networks_visited[i] = 0; ++ // If the network has any subnets, we will break it into smaller parts ++ // without the subnets. ++ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets); ++ if (!excluded) { ++ loc_network_list_unref(subnets); ++ return -1; ++ } ++ ++ // Merge subnets onto the stack ++ r = loc_network_list_merge(enumerator->stack, subnets); ++ if (r) { ++ loc_network_list_unref(subnets); ++ loc_network_list_unref(excluded); ++ ++ return r; ++ } ++ ++ // Push excluded list onto the stack ++ r = loc_network_list_merge(enumerator->stack, excluded); ++ if (r) { ++ loc_network_list_unref(subnets); ++ loc_network_list_unref(excluded); ++ ++ return r; ++ } ++ ++ loc_network_list_unref(subnets); ++ loc_network_list_unref(excluded); ++ ++ // Replace network with the first one from the stack ++ loc_network_unref(*network); ++ *network = loc_network_list_pop_first(enumerator->stack); -- if not country in writers: -- continue -- -- gaps = [ -- subnet for subnet in subnets -- if not subnet.has_flag(flag) -- ] -- -- writers[country].write(network, gaps) -- -- # Push all subnets back onto the stack -- for subnet in reversed(subnets): -- networks.push(subnet) -+ try: -+ writers[country].write(network) -+ except KeyError: -+ pass - - # Write everything to the filesystem - for writer in writers.values(): -@@ -298,33 +222,3 @@ class Exporter(object): - ) + return 0; + } - return os.path.join(directory, filename) -- -- --class BufferedStack(object): -- """ -- This class takes an iterator and when being iterated -- over it returns objects from that iterator for as long -- as there are any. -- -- It additionally has a function to put an item back on -- the back so that it will be returned again at the next -- iteration. -- """ -- def __init__(self, iterator): -- self.iterator = iterator -- self.stack = [] -- -- def __iter__(self): -- return self -- -- def __next__(self): -- if self.stack: -- return self.stack.pop(0) -- -- return next(self.iterator) -- -- def push(self, elem): -- """ -- Takes an element and puts it on the stack -- """ -- self.stack.insert(0, elem) --- -2.20.1 - -From e0b9ff5f38beb0d560b16db881647e5a75127df1 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Sun, 15 Nov 2020 15:02:28 +0000 -Subject: [PATCH 055/111] Move network lists into an own file - -Signed-off-by: Michael Tremer ---- - Makefile.am | 2 + - src/libloc.sym | 1 + - src/loc/network-list.h | 37 +++++++ - src/loc/network.h | 20 +--- - src/network-list.c | 224 +++++++++++++++++++++++++++++++++++++++++ - src/network.c | 207 +------------------------------------ - src/python/network.c | 1 + - 7 files changed, 269 insertions(+), 223 deletions(-) - create mode 100644 src/loc/network-list.h - create mode 100644 src/network-list.c - -diff --git a/Makefile.am b/Makefile.am -index a0431a6..f0d8c4c 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -96,6 +96,7 @@ pkginclude_HEADERS = \ - src/loc/database.h \ - src/loc/format.h \ - src/loc/network.h \ -+ src/loc/network-list.h \ - src/loc/private.h \ - src/loc/stringpool.h \ - src/loc/resolv.h \ -@@ -110,6 +111,7 @@ src_libloc_la_SOURCES = \ - src/country.c \ - src/database.c \ - src/network.c \ -+ src/network-list.c \ - src/resolv.c \ - src/stringpool.c \ - src/writer.c ++LOC_EXPORT int loc_database_enumerator_next_network( ++ struct loc_database_enumerator* enumerator, struct loc_network** network) { ++ // Do not do anything if not in network mode ++ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) ++ return 0; ++ ++ // Flatten output? ++ if (enumerator->flatten) ++ return __loc_database_enumerator_next_network_flattened(enumerator, network); ++ ++ return __loc_database_enumerator_next_network(enumerator, network, 1); ++} ++ + LOC_EXPORT int loc_database_enumerator_next_country( + struct loc_database_enumerator* enumerator, struct loc_country** country) { + *country = NULL; diff --git a/src/libloc.sym b/src/libloc.sym -index 6139db6..453a1be 100644 +index b8296eb..ee333f1 100644 --- a/src/libloc.sym +++ b/src/libloc.sym -@@ -87,6 +87,7 @@ global: +@@ -37,6 +37,18 @@ global: + loc_as_set_name; + loc_as_unref; + ++ # AS List ++ loc_as_list_append; ++ loc_as_list_clear; ++ loc_as_list_contains; ++ loc_as_list_contains_number; ++ loc_as_list_empty; ++ loc_as_list_get; ++ loc_as_list_new; ++ loc_as_list_ref; ++ loc_as_list_size; ++ loc_as_list_unref; ++ + # Country + loc_country_cmp; + loc_country_code_is_valid; +@@ -49,6 +61,18 @@ global: + loc_country_set_name; + loc_country_unref; + ++ # Country List ++ loc_country_list_append; ++ loc_country_list_clear; ++ loc_country_list_contains; ++ loc_country_list_contains_code; ++ loc_country_list_empty; ++ loc_country_list_get; ++ loc_country_list_new; ++ loc_country_list_ref; ++ loc_country_list_size; ++ loc_country_list_unref; ++ + # Database + loc_database_add_as; + loc_database_count_as; +@@ -66,13 +90,15 @@ global: + loc_database_verify; + + # Database Enumerator ++ loc_database_enumerator_get_asns; ++ loc_database_enumerator_get_countries; + loc_database_enumerator_new; + loc_database_enumerator_next_as; + loc_database_enumerator_next_country; + loc_database_enumerator_next_network; + loc_database_enumerator_ref; +- loc_database_enumerator_set_asn; +- loc_database_enumerator_set_country_code; ++ loc_database_enumerator_set_asns; ++ loc_database_enumerator_set_countries; + loc_database_enumerator_set_family; + loc_database_enumerator_set_flag; + loc_database_enumerator_set_string; +@@ -80,24 +106,48 @@ global: + + # Network + loc_network_address_family; ++ loc_network_cmp; ++ loc_network_exclude; ++ loc_network_exclude_list; + loc_network_format_first_address; loc_network_format_last_address; loc_network_get_asn; loc_network_get_country_code; -+ loc_network_gt; ++ loc_network_get_first_address; ++ loc_network_get_last_address; loc_network_has_flag; - loc_network_is_subnet; - loc_network_is_subnet_of; -diff --git a/src/loc/network-list.h b/src/loc/network-list.h +- loc_network_is_subnet_of; ++ loc_network_is_subnet; ++ loc_network_match_address; + loc_network_match_asn; + loc_network_match_country_code; + loc_network_match_flag; + loc_network_new; + loc_network_new_from_string; ++ loc_network_overlaps; ++ loc_network_prefix; + loc_network_ref; + loc_network_set_asn; + loc_network_set_country_code; + loc_network_set_flag; + loc_network_str; ++ loc_network_subnets; + loc_network_unref; + ++ # Network List ++ loc_network_list_clear; ++ loc_network_list_contains; ++ loc_network_list_dump; ++ loc_network_list_empty; ++ loc_network_list_get; ++ loc_network_list_merge; ++ loc_network_list_new; ++ loc_network_list_pop; ++ loc_network_list_pop_first; ++ loc_network_list_push; ++ loc_network_list_ref; ++ loc_network_list_size; ++ loc_network_list_unref; ++ + # Writer + loc_writer_add_as; + loc_writer_add_country; +diff --git a/src/loc/as-list.h b/src/loc/as-list.h new file mode 100644 -index 0000000..af3b28d +index 0000000..7b5c4e8 --- /dev/null -+++ b/src/loc/network-list.h -@@ -0,0 +1,37 @@ ++++ b/src/loc/as-list.h +@@ -0,0 +1,41 @@ +/* + libloc - A library to determine the location of someone on the Internet + -+ Copyright (C) 2020 IPFire Development Team ++ Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public @@ -5294,82 +1137,41 @@ index 0000000..af3b28d + Lesser General Public License for more details. +*/ + -+#ifndef LIBLOC_NETWORK_LIST_H -+#define LIBLOC_NETWORK_LIST_H ++#ifndef LIBLOC_AS_LIST_H ++#define LIBLOC_AS_LIST_H + -+struct loc_network_list; -+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); -+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); -+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); -+size_t loc_network_list_size(struct loc_network_list* list); -+int loc_network_list_empty(struct loc_network_list* list); -+void loc_network_list_clear(struct loc_network_list* list); -+void loc_network_list_dump(struct loc_network_list* list); -+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); -+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); -+struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); -+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); -+void loc_network_list_sort(struct loc_network_list* list); -+void loc_network_list_reverse(struct loc_network_list* list); -+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); ++#include ++#include ++ ++struct loc_as_list; ++ ++int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list); ++struct loc_as_list* loc_as_list_ref(struct loc_as_list* list); ++struct loc_as_list* loc_as_list_unref(struct loc_as_list* list); ++ ++size_t loc_as_list_size(struct loc_as_list* list); ++int loc_as_list_empty(struct loc_as_list* list); ++void loc_as_list_clear(struct loc_as_list* list); ++ ++struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index); ++int loc_as_list_append(struct loc_as_list* list, struct loc_as* as); ++ ++int loc_as_list_contains( ++ struct loc_as_list* list, struct loc_as* as); ++int loc_as_list_contains_number( ++ struct loc_as_list* list, uint32_t number); + +#endif -diff --git a/src/loc/network.h b/src/loc/network.h -index 203e61c..d86b685 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -21,6 +21,7 @@ - - #include - #include -+#include - - enum loc_network_flags { - LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1 -@@ -55,6 +56,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_eq(struct loc_network* self, struct loc_network* other); -+int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); -@@ -64,24 +66,6 @@ struct loc_network_list* loc_network_exclude( - struct loc_network_list* loc_network_exclude_list( - struct loc_network* network, struct loc_network_list* list); - --// List --struct loc_network_list; --int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); --struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); --struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); --size_t loc_network_list_size(struct loc_network_list* list); --int loc_network_list_empty(struct loc_network_list* list); --void loc_network_list_clear(struct loc_network_list* list); --void loc_network_list_dump(struct loc_network_list* list); --struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); --int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); --struct loc_network* loc_network_list_pop(struct loc_network_list* list); --struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); --int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); --void loc_network_list_sort(struct loc_network_list* list); --void loc_network_list_reverse(struct loc_network_list* list); --int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); -- - #ifdef LIBLOC_PRIVATE - - int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj); -diff --git a/src/network-list.c b/src/network-list.c +diff --git a/src/loc/country-list.h b/src/loc/country-list.h new file mode 100644 -index 0000000..1f6e80e +index 0000000..a7f818a --- /dev/null -+++ b/src/network-list.c -@@ -0,0 +1,224 @@ ++++ b/src/loc/country-list.h +@@ -0,0 +1,43 @@ +/* + libloc - A library to determine the location of someone on the Internet + -+ Copyright (C) 2020 IPFire Development Team ++ Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public @@ -5378,511 +1180,166 @@ index 0000000..1f6e80e + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#include -+#include -+ -+#include -+#include -+#include -+ -+struct loc_network_list { -+ struct loc_ctx* ctx; -+ int refcount; -+ -+ struct loc_network* list[1024]; -+ size_t size; -+ size_t max_size; -+}; -+ -+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, -+ struct loc_network_list** list) { -+ struct loc_network_list* l = calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; -+ -+ l->ctx = loc_ref(ctx); -+ l->refcount = 1; -+ -+ // Do not allow this list to grow larger than this -+ l->max_size = 1024; -+ -+ DEBUG(l->ctx, "Network list allocated at %p\n", l); -+ *list = l; -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { -+ list->refcount++; -+ -+ return list; -+} -+ -+static void loc_network_list_free(struct loc_network_list* list) { -+ DEBUG(list->ctx, "Releasing network list at %p\n", list); -+ -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_network_unref(list->list[i]); -+ -+ loc_unref(list->ctx); -+ free(list); -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { -+ if (!list) -+ return NULL; -+ -+ if (--list->refcount > 0) -+ return list; -+ -+ loc_network_list_free(list); -+ return NULL; -+} -+ -+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { -+ return list->size; -+} -+ -+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { -+ return list->size == 0; -+} -+ -+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_network_unref(list->list[i]); -+ -+ list->size = 0; -+} -+ -+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { -+ struct loc_network* network; -+ char* s; -+ -+ for (unsigned int i = 0; i < list->size; i++) { -+ network = list->list[i]; -+ -+ s = loc_network_str(network); -+ -+ INFO(list->ctx, "%s\n", s); -+ free(s); -+ } -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { -+ // Check index -+ if (index >= list->size) -+ return NULL; -+ -+ return loc_network_ref(list->list[index]); -+} -+ -+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -+ // Do not add networks that are already on the list -+ if (loc_network_list_contains(list, network)) -+ return 0; -+ -+ // Check if we have space left -+ if (list->size == list->max_size) { -+ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); -+ return -ENOMEM; -+ } -+ -+ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); -+ -+ list->list[list->size++] = loc_network_ref(network); -+ -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); -+ return NULL; -+ } -+ -+ struct loc_network* network = list->list[--list->size]; -+ -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -+ -+ return network; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); -+ return NULL; -+ } -+ -+ struct loc_network* network = list->list[0]; ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ + -+ // Move all elements to the top of the stack -+ for (unsigned int i = 0; i < --list->size; i++) { -+ list->list[i] = list->list[i+1]; -+ } ++#ifndef LIBLOC_COUNTRY_LIST_H ++#define LIBLOC_COUNTRY_LIST_H + -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++#include + -+ return network; -+} ++#include ++#include + -+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { -+ for (unsigned int i = 0; i < list->size; i++) { -+ if (loc_network_eq(list->list[i], network)) -+ return 1; -+ } ++struct loc_country_list; + -+ return 0; -+} ++int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list); ++struct loc_country_list* loc_country_list_ref(struct loc_country_list* list); ++struct loc_country_list* loc_country_list_unref(struct loc_country_list* list); + -+static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { -+ // Do nothing for invalid indices -+ if (i1 >= list->size || i2 >= list->size) -+ return; ++size_t loc_country_list_size(struct loc_country_list* list); ++int loc_country_list_empty(struct loc_country_list* list); ++void loc_country_list_clear(struct loc_country_list* list); + -+ struct loc_network* network1 = list->list[i1]; -+ struct loc_network* network2 = list->list[i2]; ++struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index); ++int loc_country_list_append(struct loc_country_list* list, struct loc_country* country); + -+ list->list[i1] = network2; -+ list->list[i2] = network1; -+} ++int loc_country_list_contains( ++ struct loc_country_list* list, struct loc_country* country); ++int loc_country_list_contains_code( ++ struct loc_country_list* list, const char* code); + -+LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { -+ unsigned int i = 0; -+ unsigned int j = list->size - 1; ++#endif +diff --git a/src/loc/database.h b/src/loc/database.h +index 43173dd..70801f0 100644 +--- a/src/loc/database.h ++++ b/src/loc/database.h +@@ -25,6 +25,7 @@ + #include + #include + #include ++#include + + struct loc_database; + int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f); +@@ -55,15 +56,24 @@ enum loc_database_enumerator_mode { + LOC_DB_ENUMERATE_COUNTRIES = 3, + }; + ++enum loc_database_enumerator_flags { ++ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0), ++}; + -+ while (i < j) { -+ loc_network_list_swap(list, i++, j--); -+ } -+} + struct loc_database_enumerator; + int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, +- struct loc_database* db, enum loc_database_enumerator_mode mode); ++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags); + struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator); + struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); + + int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string); +-int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code); +-int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn); ++struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); ++int loc_database_enumerator_set_countries( ++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries); ++struct loc_as_list* loc_database_enumerator_get_asns( ++ struct loc_database_enumerator* enumerator); ++int loc_database_enumerator_set_asns( ++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns); + int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); + int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); + int loc_database_enumerator_next_as( +diff --git a/src/loc/network-list.h b/src/loc/network-list.h +new file mode 100644 +index 0000000..bee21c4 +--- /dev/null ++++ b/src/loc/network-list.h +@@ -0,0 +1,37 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet + -+LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { -+ unsigned int n = list->size; -+ int swapped; ++ Copyright (C) 2020 IPFire Development Team + -+ do { -+ swapped = 0; ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. + -+ for (unsigned int i = 1; i < n; i++) { -+ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { -+ loc_network_list_swap(list, i-1, i); -+ swapped = 1; -+ } -+ } ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ + -+ n--; -+ } while (swapped); -+} ++#ifndef LIBLOC_NETWORK_LIST_H ++#define LIBLOC_NETWORK_LIST_H + -+LOC_EXPORT int loc_network_list_merge( -+ struct loc_network_list* self, struct loc_network_list* other) { -+ int r; ++#include + -+ for (unsigned int i = 0; i < other->size; i++) { -+ r = loc_network_list_push(self, other->list[i]); -+ if (r) -+ return r; -+ } ++struct loc_network_list; ++int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); ++struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); ++struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); ++size_t loc_network_list_size(struct loc_network_list* list); ++int loc_network_list_empty(struct loc_network_list* list); ++void loc_network_list_clear(struct loc_network_list* list); ++void loc_network_list_dump(struct loc_network_list* list); ++struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); ++int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); ++struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); ++int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); ++int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); + -+ return 0; -+} -diff --git a/src/network.c b/src/network.c -index d015579..28ca2df 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -29,6 +29,7 @@ - #include - #include - #include ++#endif +diff --git a/src/loc/network.h b/src/loc/network.h +index 70c3803..af3dafd 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -21,6 +21,7 @@ + + #include + #include +#include - #include - struct loc_network { -@@ -436,7 +437,7 @@ LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* othe - return 1; - } + enum loc_network_flags { + LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1 +@@ -37,8 +38,11 @@ struct loc_network* loc_network_ref(struct loc_network* network); + struct loc_network* loc_network_unref(struct loc_network* network); + char* loc_network_str(struct loc_network* network); + int loc_network_address_family(struct loc_network* network); ++unsigned int loc_network_prefix(struct loc_network* network); --static int loc_network_gt(struct loc_network* self, struct loc_network* other) { -+LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) { - // Families must match - if (self->family != other->family) - return -1; -@@ -1106,207 +1107,3 @@ int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { - struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { - return loc_network_ref(node->network); - } -- --// List -- --struct loc_network_list { -- struct loc_ctx* ctx; -- int refcount; -- -- struct loc_network* list[1024]; -- size_t size; -- size_t max_size; --}; -- --LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, -- struct loc_network_list** list) { -- struct loc_network_list* l = calloc(1, sizeof(*l)); -- if (!l) -- return -ENOMEM; -- -- l->ctx = loc_ref(ctx); -- l->refcount = 1; -- -- // Do not allow this list to grow larger than this -- l->max_size = 1024; -- -- DEBUG(l->ctx, "Network list allocated at %p\n", l); -- *list = l; -- return 0; --} -- --LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { -- list->refcount++; -- -- return list; --} -- --static void loc_network_list_free(struct loc_network_list* list) { -- DEBUG(list->ctx, "Releasing network list at %p\n", list); -- -- for (unsigned int i = 0; i < list->size; i++) -- loc_network_unref(list->list[i]); -- -- loc_unref(list->ctx); -- free(list); --} -- --LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { -- if (!list) -- return NULL; -- -- if (--list->refcount > 0) -- return list; -- -- loc_network_list_free(list); -- return NULL; --} -- --LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { -- return list->size; --} -- --LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { -- return list->size == 0; --} -- --LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { -- for (unsigned int i = 0; i < list->size; i++) -- loc_network_unref(list->list[i]); -- -- list->size = 0; --} -- --LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { -- struct loc_network* network; -- char* s; -- -- for (unsigned int i = 0; i < list->size; i++) { -- network = list->list[i]; -- -- s = loc_network_str(network); -- -- INFO(list->ctx, "%s\n", s); -- free(s); -- } --} -- --LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { -- // Check index -- if (index >= list->size) -- return NULL; -- -- return loc_network_ref(list->list[index]); --} -- --LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -- // Do not add networks that are already on the list -- if (loc_network_list_contains(list, network)) -- return 0; -- -- // Check if we have space left -- if (list->size == list->max_size) { -- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); -- return -ENOMEM; -- } -- -- DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); -- -- list->list[list->size++] = loc_network_ref(network); -- -- return 0; --} -- --LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { -- // Return nothing when empty -- if (loc_network_list_empty(list)) { -- DEBUG(list->ctx, "%p: Popped empty stack\n", list); -- return NULL; -- } -- -- struct loc_network* network = list->list[--list->size]; -- -- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -- -- return network; --} -- --LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { -- // Return nothing when empty -- if (loc_network_list_empty(list)) { -- DEBUG(list->ctx, "%p: Popped empty stack\n", list); -- return NULL; -- } -- -- struct loc_network* network = list->list[0]; -- -- // Move all elements to the top of the stack -- for (unsigned int i = 0; i < --list->size; i++) { -- list->list[i] = list->list[i+1]; -- } -- -- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -- -- return network; --} -- --LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { -- for (unsigned int i = 0; i < list->size; i++) { -- if (loc_network_eq(list->list[i], network)) -- return 1; -- } -- -- return 0; --} -- --static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { -- // Do nothing for invalid indices -- if (i1 >= list->size || i2 >= list->size) -- return; -- -- struct loc_network* network1 = list->list[i1]; -- struct loc_network* network2 = list->list[i2]; -- -- list->list[i1] = network2; -- list->list[i2] = network1; --} -- --LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { -- unsigned int i = 0; -- unsigned int j = list->size - 1; -- -- while (i < j) { -- loc_network_list_swap(list, i++, j--); -- } --} -- --LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { -- unsigned int n = list->size; -- int swapped; -- -- do { -- swapped = 0; -- -- for (unsigned int i = 1; i < n; i++) { -- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { -- loc_network_list_swap(list, i-1, i); -- swapped = 1; -- } -- } -- -- n--; -- } while (swapped); --} -- --LOC_EXPORT int loc_network_list_merge( -- struct loc_network_list* self, struct loc_network_list* other) { -- int r; -- -- for (unsigned int i = 0; i < other->size; i++) { -- r = loc_network_list_push(self, other->list[i]); -- if (r) -- return r; -- } -- -- return 0; --} -diff --git a/src/python/network.c b/src/python/network.c -index 11f672b..ed91d65 100644 ---- a/src/python/network.c -+++ b/src/python/network.c -@@ -20,6 +20,7 @@ ++const struct in6_addr* loc_network_get_first_address(struct loc_network* network); + char* loc_network_format_first_address(struct loc_network* network); ++const struct in6_addr* loc_network_get_last_address(struct loc_network* network); + char* loc_network_format_last_address(struct loc_network* network); + int loc_network_match_address(struct loc_network* network, const struct in6_addr* address); - #include - #include -+#include +@@ -54,7 +58,14 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag); + int loc_network_set_flag(struct loc_network* network, uint32_t flag); + int loc_network_match_flag(struct loc_network* network, uint32_t flag); + +-int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); ++int loc_network_cmp(struct loc_network* self, struct loc_network* other); ++int loc_network_overlaps(struct loc_network* self, struct loc_network* other); ++int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); ++int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2); ++struct loc_network_list* loc_network_exclude( ++ struct loc_network* self, struct loc_network* other); ++struct loc_network_list* loc_network_exclude_list( ++ struct loc_network* network, struct loc_network_list* list); + + #ifdef LIBLOC_PRIVATE - #include "locationmodule.h" - #include "network.h" --- -2.20.1 - -From e646a8f35ec7eff009414b3fd107c9af5cf39a86 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Mon, 16 Nov 2020 15:13:28 +0000 -Subject: [PATCH 056/111] Implement filtering for multiple countries in the - enumerator - -This will allow us to speed up the export of the database -if only a few countries should be returned. - -Signed-off-by: Michael Tremer ---- - Makefile.am | 2 + - src/country-list.c | 138 +++++++++++++++++++++++++++++++++++++++++ - src/country.c | 3 + - src/database.c | 47 ++++++-------- - src/libloc.sym | 15 ++++- - src/loc/country-list.h | 43 +++++++++++++ - src/loc/database.h | 5 +- - src/python/database.c | 57 ++++++++++++++--- - 8 files changed, 274 insertions(+), 36 deletions(-) - create mode 100644 src/country-list.c - create mode 100644 src/loc/country-list.h - -diff --git a/Makefile.am b/Makefile.am -index f0d8c4c..f4ca3c8 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -93,6 +93,7 @@ pkginclude_HEADERS = \ - src/loc/as.h \ - src/loc/compat.h \ - src/loc/country.h \ -+ src/loc/country-list.h \ - src/loc/database.h \ - src/loc/format.h \ - src/loc/network.h \ -@@ -109,6 +110,7 @@ src_libloc_la_SOURCES = \ - src/libloc.c \ - src/as.c \ - src/country.c \ -+ src/country-list.c \ - src/database.c \ - src/network.c \ - src/network-list.c \ -diff --git a/src/country-list.c b/src/country-list.c +diff --git a/src/network-list.c b/src/network-list.c new file mode 100644 -index 0000000..ae0d71a +index 0000000..e2b20f3 --- /dev/null -+++ b/src/country-list.c -@@ -0,0 +1,138 @@ ++++ b/src/network-list.c +@@ -0,0 +1,295 @@ +/* + libloc - A library to determine the location of someone on the Internet + @@ -5901,2068 +1358,1139 @@ index 0000000..ae0d71a + +#include +#include ++#include + -+#include -+#include ++#include ++#include +#include + -+struct loc_country_list { ++struct loc_network_list { + struct loc_ctx* ctx; + int refcount; + -+ struct loc_country* list[1024]; ++ struct loc_network** elements; ++ size_t elements_size; ++ + size_t size; -+ size_t max_size; +}; + -+LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, -+ struct loc_country_list** list) { -+ struct loc_country_list* l = calloc(1, sizeof(*l)); ++static int loc_network_list_grow(struct loc_network_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); ++ ++ struct loc_network** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; ++ ++ list->elements = elements; ++ list->elements_size += size; ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, ++ struct loc_network_list** list) { ++ struct loc_network_list* l = calloc(1, sizeof(*l)); + if (!l) + return -ENOMEM; + + l->ctx = loc_ref(ctx); + l->refcount = 1; + -+ // Do not allow this list to grow larger than this -+ l->max_size = 1024; -+ -+ DEBUG(l->ctx, "Country list allocated at %p\n", l); ++ DEBUG(l->ctx, "Network list allocated at %p\n", l); + *list = l; -+ + return 0; +} + -+LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) { ++LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { + list->refcount++; + + return list; +} + -+static void loc_country_list_free(struct loc_country_list* list) { -+ DEBUG(list->ctx, "Releasing country list at %p\n", list); ++static void loc_network_list_free(struct loc_network_list* list) { ++ DEBUG(list->ctx, "Releasing network list at %p\n", list); + -+ loc_country_list_clear(list); ++ // Remove all content ++ loc_network_list_clear(list); + + loc_unref(list->ctx); + free(list); +} + -+LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) { ++LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { + if (!list) + return NULL; + + if (--list->refcount > 0) + return list; + -+ loc_country_list_free(list); ++ loc_network_list_free(list); + return NULL; +} + -+LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) { ++LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { + return list->size; +} + -+LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { ++LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { + return list->size == 0; +} + -+LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_country_unref(list->list[i]); -+} -+ -+LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { -+ // Check index -+ if (index >= list->size) -+ return NULL; -+ -+ return loc_country_ref(list->list[index]); -+} -+ -+LOC_EXPORT int loc_country_list_append( -+ struct loc_country_list* list, struct loc_country* country) { -+ if (loc_country_list_contains(list, country)) -+ return 0; -+ -+ // Check if we have space left -+ if (list->size == list->max_size) { -+ ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list); -+ return -ENOMEM; -+ } -+ -+ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); -+ -+ list->list[list->size++] = loc_country_ref(country); -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_country_list_contains( -+ struct loc_country_list* list, struct loc_country* country) { -+ for (unsigned int i = 0; i < list->size; i++) { -+ if (loc_country_cmp(country, list->list[i]) == 0) -+ return 1; -+ } -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_country_list_contains_code( -+ struct loc_country_list* list, const char* code) { -+ struct loc_country* country; -+ -+ int r = loc_country_new(list->ctx, &country, code); -+ if (r) -+ return -1; -+ -+ r = loc_country_list_contains(list, country); -+ loc_country_unref(country); -+ -+ return r; -+} -diff --git a/src/country.c b/src/country.c -index 2ba93e6..7aac0db 100644 ---- a/src/country.c -+++ b/src/country.c -@@ -34,6 +34,9 @@ struct loc_country { - }; - - LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) { -+ if (!loc_country_code_is_valid(country_code)) -+ return -EINVAL; -+ - struct loc_country* c = calloc(1, sizeof(*c)); - if (!c) - return -ENOMEM; -diff --git a/src/database.c b/src/database.c -index b9d870f..29823b2 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -40,6 +40,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -99,7 +100,7 @@ struct loc_database_enumerator { - - // Search string - char* string; -- char country_code[3]; -+ struct loc_country_list* countries; - uint32_t asn; - enum loc_network_flags flags; - int family; -@@ -1017,33 +1018,20 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator - return 0; - } - --LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) { -- // Set empty country code -- if (!country_code || !*country_code) { -- *enumerator->country_code = '\0'; -- return 0; -- } -+LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries( -+ struct loc_database_enumerator* enumerator) { -+ if (!enumerator->countries) -+ return NULL; - -- // Treat A1, A2, A3 as special country codes, -- // but perform search for flags instead -- if (strcmp(country_code, "A1") == 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_ANONYMOUS_PROXY); -- } else if (strcmp(country_code, "A2") == 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_SATELLITE_PROVIDER); -- } else if (strcmp(country_code, "A3") == 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_ANYCAST); -- } -+ return loc_country_list_ref(enumerator->countries); -+} - -- // Country codes must be two characters -- if (!loc_country_code_is_valid(country_code)) -- return -EINVAL; -+LOC_EXPORT int loc_database_enumerator_set_countries( -+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) { -+ if (enumerator->countries) -+ loc_country_list_unref(enumerator->countries); - -- for (unsigned int i = 0; i < 3; i++) { -- enumerator->country_code[i] = country_code[i]; -- } -+ enumerator->countries = loc_country_list_ref(countries); - - return 0; - } -@@ -1129,6 +1117,12 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } - -+static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) { -+ const char* country_code = loc_network_get_country_code(network); -+ -+ return loc_country_list_contains_code(countries, country_code); -+} ++LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { ++ if (!list->elements) ++ return; + - static int loc_database_enumerator_filter_network( - struct loc_database_enumerator* enumerator, struct loc_network* network) { - // Skip if the family does not match -@@ -1136,8 +1130,7 @@ static int loc_database_enumerator_filter_network( - return 1; - - // Skip if the country code does not match -- if (*enumerator->country_code && -- !loc_network_match_country_code(network, enumerator->country_code)) -+ if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries)) - return 1; - - // Skip if the ASN does not match -diff --git a/src/libloc.sym b/src/libloc.sym -index 453a1be..40e9f88 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -49,6 +49,18 @@ global: - loc_country_set_name; - loc_country_unref; - -+ # Country List -+ loc_country_list_append; -+ loc_country_list_clear; -+ loc_country_list_contains; -+ loc_country_list_contains_code; -+ loc_country_list_empty; -+ loc_country_list_get; -+ loc_country_list_new; -+ loc_country_list_ref; -+ loc_country_list_size; -+ loc_country_list_unref; ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_network_unref(list->elements[i]); + - # Database - loc_database_add_as; - loc_database_count_as; -@@ -66,13 +78,14 @@ global: - loc_database_verify; - - # Database Enumerator -+ loc_database_enumerator_get_countries; - loc_database_enumerator_new; - loc_database_enumerator_next_as; - loc_database_enumerator_next_country; - loc_database_enumerator_next_network; - loc_database_enumerator_ref; - loc_database_enumerator_set_asn; -- loc_database_enumerator_set_country_code; -+ loc_database_enumerator_set_countries; - loc_database_enumerator_set_family; - loc_database_enumerator_set_flag; - loc_database_enumerator_set_string; -diff --git a/src/loc/country-list.h b/src/loc/country-list.h -new file mode 100644 -index 0000000..a7f818a ---- /dev/null -+++ b/src/loc/country-list.h -@@ -0,0 +1,43 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet ++ free(list->elements); ++ list->elements = NULL; ++ list->elements_size = 0; + -+ Copyright (C) 2017 IPFire Development Team ++ list->size = 0; ++} + -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. ++LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { ++ struct loc_network* network; ++ char* s; + -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ ++ for (unsigned int i = 0; i < list->size; i++) { ++ network = list->elements[i]; + -+#ifndef LIBLOC_COUNTRY_LIST_H -+#define LIBLOC_COUNTRY_LIST_H ++ s = loc_network_str(network); + -+#include ++ INFO(list->ctx, "%4d: %s\n", i, s); ++ free(s); ++ } ++} + -+#include -+#include ++LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; + -+struct loc_country_list; ++ return loc_network_ref(list->elements[index]); ++} + -+int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list); -+struct loc_country_list* loc_country_list_ref(struct loc_country_list* list); -+struct loc_country_list* loc_country_list_unref(struct loc_country_list* list); ++static off_t loc_network_list_find(struct loc_network_list* list, ++ struct loc_network* network, int* found) { ++ off_t lo = 0; ++ off_t hi = list->size - 1; ++ int result; + -+size_t loc_country_list_size(struct loc_country_list* list); -+int loc_country_list_empty(struct loc_country_list* list); -+void loc_country_list_clear(struct loc_country_list* list); ++ // Since we are working on an ordered list, there is often a good chance that ++ // the network we are looking for is at the end or has to go to the end. ++ if (hi >= 0) { ++ result = loc_network_cmp(network, list->elements[hi]); + -+struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index); -+int loc_country_list_append(struct loc_country_list* list, struct loc_country* country); ++ // Match, so we are done ++ if (result == 0) { ++ *found = 1; + -+int loc_country_list_contains( -+ struct loc_country_list* list, struct loc_country* country); -+int loc_country_list_contains_code( -+ struct loc_country_list* list, const char* code); ++ return hi; ++ ++ // This needs to be added after the last one ++ } else if (result > 0) { ++ *found = 0; ++ ++ return hi + 1; ++ } ++ } + ++#ifdef ENABLE_DEBUG ++ // Save start time ++ clock_t start = clock(); +#endif -diff --git a/src/loc/database.h b/src/loc/database.h -index 14eb5ea..246e5c5 100644 ---- a/src/loc/database.h -+++ b/src/loc/database.h -@@ -25,6 +25,7 @@ - #include - #include - #include -+#include - - struct loc_database; - int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f); -@@ -66,7 +67,9 @@ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_ - struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); - - int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string); --int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code); -+struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); -+int loc_database_enumerator_set_countries( -+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries); - int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn); - int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); - int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); -diff --git a/src/python/database.c b/src/python/database.c -index d169547..e6f6f37 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -258,14 +258,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { - } - - static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { -- char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL }; -- const char* country_code = NULL; -+ char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL }; -+ PyObject* country_codes = NULL; - unsigned int asn = 0; - int flags = 0; - int family = 0; - int flatten = 0; - -- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten)) -+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist, -+ &PyList_Type, &country_codes, &asn, &flags, &family, &flatten)) - return NULL; - - struct loc_database_enumerator* enumerator; -@@ -277,13 +278,55 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - } - - // Set country code we are searching for -- if (country_code) { -- r = loc_database_enumerator_set_country_code(enumerator, country_code); -- -+ if (country_codes) { -+ struct loc_country_list* countries; -+ r = loc_country_list_new(loc_ctx, &countries); - if (r) { -- PyErr_SetFromErrno(PyExc_SystemError); -+ PyErr_SetString(PyExc_SystemError, "Could not create country list"); - return NULL; - } + -+ for (unsigned int i = 0; i < PyList_Size(country_codes); i++) { -+ PyObject* item = PyList_GetItem(country_codes, i); ++ off_t i = 0; + -+ if (!PyUnicode_Check(item)) { -+ PyErr_SetString(PyExc_TypeError, "Country codes must be strings"); -+ loc_country_list_unref(countries); -+ return NULL; -+ } ++ while (lo <= hi) { ++ i = (lo + hi) / 2; + -+ const char* country_code = PyUnicode_AsUTF8(item); ++ // Check if this is a match ++ result = loc_network_cmp(network, list->elements[i]); + -+ struct loc_country* country; -+ r = loc_country_new(loc_ctx, &country, country_code); -+ if (r) { -+ if (r == -EINVAL) { -+ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code); -+ } else { -+ PyErr_SetString(PyExc_SystemError, "Could not create country"); -+ } ++ if (result == 0) { ++ *found = 1; + -+ loc_country_list_unref(countries); -+ return NULL; -+ } ++#ifdef ENABLE_DEBUG ++ clock_t end = clock(); + -+ // Append it to the list -+ r = loc_country_list_append(countries, country); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not append country to the list"); ++ // Log how fast this has been ++ DEBUG(list->ctx, "Found network in %.4fms at %jd\n", ++ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); ++#endif + -+ loc_country_list_unref(countries); -+ loc_country_unref(country); -+ return NULL; -+ } ++ return i; ++ } + -+ loc_country_unref(country); ++ if (result > 0) { ++ lo = i + 1; ++ i++; ++ } else { ++ hi = i - 1; + } ++ } + -+ loc_database_enumerator_set_countries(enumerator, countries); ++ *found = 0; + -+ Py_DECREF(country_codes); -+ loc_country_list_unref(countries); - } - - // Set the ASN we are searching for --- -2.20.1 - -From 7af51f8a579c79714992a3e175036fb511139310 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Mon, 16 Nov 2020 15:20:50 +0000 -Subject: [PATCH 057/111] python: Only return country codes we want - -Signed-off-by: Michael Tremer ---- - src/python/export.py | 8 +++++++- - 1 file changed, 7 insertions(+), 1 deletion(-) - -diff --git a/src/python/export.py b/src/python/export.py -index be4a68e..5e7fe53 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -184,8 +184,14 @@ class Exporter(object): - - writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) - -+ # Filter countries from special country codes -+ country_codes = [ -+ country_code for country_code in countries if not country_code in flags.values() -+ ] ++#ifdef ENABLE_DEBUG ++ clock_t end = clock(); + - # Get all networks that match the family -- networks = self.db.search_networks(family=family, flatten=True) -+ networks = self.db.search_networks(family=family, -+ country_codes=country_codes, flatten=True) - - # Walk through all networks - for network in networks: --- -2.20.1 - -From bd1dc6bf6fe4ce40bf12e7426e283b31afd274e1 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Mon, 16 Nov 2020 15:25:15 +0000 -Subject: [PATCH 058/111] database: Filter flags in C - -Signed-off-by: Michael Tremer ---- - src/python/export.py | 16 +++++++++++----- - 1 file changed, 11 insertions(+), 5 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index 5e7fe53..739742f 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -29,7 +29,7 @@ import _location - log = logging.getLogger("location.export") - log.propagate = 1 - --flags = { -+FLAGS = { - _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", - _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", - _location.NETWORK_FLAG_ANYCAST : "A3", -@@ -186,12 +186,18 @@ class Exporter(object): - - # Filter countries from special country codes - country_codes = [ -- country_code for country_code in countries if not country_code in flags.values() -+ country_code for country_code in countries if not country_code in FLAGS.values() - ] - -+ # Collect flags -+ flags = 0 -+ for flag in FLAGS: -+ if FLAGS[flag] in countries: -+ flags |= flag ++ // Log how fast this has been ++ DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n", ++ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); ++#endif + - # Get all networks that match the family - networks = self.db.search_networks(family=family, -- country_codes=country_codes, flatten=True) -+ country_codes=country_codes, flags=flags, flatten=True) - - # Walk through all networks - for network in networks: -@@ -208,10 +214,10 @@ class Exporter(object): - pass - - # Handle flags -- for flag in flags: -+ for flag in FLAGS: - if network.has_flag(flag): - # Fetch the "fake" country code -- country = flags[flag] -+ country = FLAGS[flag] - - try: - writers[country].write(network) --- -2.20.1 - -From 84a2f0c2d9cbf8ae4225802c29ccba86561c77ed Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 17 Nov 2020 16:46:48 +0000 -Subject: [PATCH 059/111] as: Add list for easier processing - -Signed-off-by: Michael Tremer ---- - Makefile.am | 2 + - src/as-list.c | 138 ++++++++++++++++++++++++++++++++++++++++++ - src/database.c | 29 +++++++-- - src/libloc.sym | 15 ++++- - src/loc/as-list.h | 41 +++++++++++++ - src/loc/database.h | 5 +- - src/python/database.c | 58 ++++++++++++++++-- - src/python/export.py | 2 +- - 8 files changed, 275 insertions(+), 15 deletions(-) - create mode 100644 src/as-list.c - create mode 100644 src/loc/as-list.h - -diff --git a/Makefile.am b/Makefile.am -index f4ca3c8..d0cc793 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -91,6 +91,7 @@ EXTRA_DIST += \ - pkginclude_HEADERS = \ - src/loc/libloc.h \ - src/loc/as.h \ -+ src/loc/as-list.h \ - src/loc/compat.h \ - src/loc/country.h \ - src/loc/country-list.h \ -@@ -109,6 +110,7 @@ lib_LTLIBRARIES = \ - src_libloc_la_SOURCES = \ - src/libloc.c \ - src/as.c \ -+ src/as-list.c \ - src/country.c \ - src/country-list.c \ - src/database.c \ -diff --git a/src/as-list.c b/src/as-list.c -new file mode 100644 -index 0000000..7c69eb0 ---- /dev/null -+++ b/src/as-list.c -@@ -0,0 +1,138 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet ++ return i; ++} ++ ++LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { ++ int found = 0; ++ ++ off_t index = loc_network_list_find(list, network, &found); ++ ++ // The network has been found on the list. Nothing to do. ++ if (found) ++ return 0; ++ ++ DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n", ++ list, network, (intmax_t)index); + -+ Copyright (C) 2020 IPFire Development Team ++ // Check if we have space left ++ if (list->size >= list->elements_size) { ++ int r = loc_network_list_grow(list, 64); ++ if (r) ++ return r; ++ } + -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. ++ // The list is now larger ++ list->size++; + -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ ++ // Move all elements out of the way ++ for (unsigned int i = list->size - 1; i > index; i--) ++ list->elements[i] = list->elements[i - 1]; + -+#include -+#include ++ // Add the new element at the right place ++ list->elements[index] = loc_network_ref(network); + -+#include -+#include -+#include ++ return 0; ++} + -+struct loc_as_list { -+ struct loc_ctx* ctx; -+ int refcount; ++LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); ++ return NULL; ++ } + -+ struct loc_as* list[1024]; -+ size_t size; -+ size_t max_size; -+}; ++ struct loc_network* network = list->elements[--list->size]; + -+LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, -+ struct loc_as_list** list) { -+ struct loc_as_list* l = calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); + -+ l->ctx = loc_ref(ctx); -+ l->refcount = 1; ++ return network; ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); ++ return NULL; ++ } + -+ // Do not allow this list to grow larger than this -+ l->max_size = 1024; ++ struct loc_network* network = list->elements[0]; + -+ DEBUG(l->ctx, "AS list allocated at %p\n", l); -+ *list = l; ++ // Move all elements to the top of the stack ++ for (unsigned int i = 0; i < list->size - 1; i++) { ++ list->elements[i] = list->elements[i+1]; ++ } ++ ++ // The list is shorter now ++ --list->size; ++ ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++ ++ return network; ++} ++ ++LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { ++ int found = 0; ++ ++ loc_network_list_find(list, network, &found); ++ ++ return found; ++} ++ ++LOC_EXPORT int loc_network_list_merge( ++ struct loc_network_list* self, struct loc_network_list* other) { ++ int r; ++ ++ for (unsigned int i = 0; i < other->size; i++) { ++ r = loc_network_list_push(self, other->elements[i]); ++ if (r) ++ return r; ++ } + + return 0; +} +diff --git a/src/network.c b/src/network.c +index 366caa2..a6b679c 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -29,6 +29,7 @@ + #include + #include + #include ++#include + #include + + struct loc_network { +@@ -97,6 +98,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s + return a; + } + ++static struct in6_addr address_increment(const struct in6_addr* address) { ++ struct in6_addr a = *address; + -+LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) { -+ list->refcount++; ++ for (int octet = 15; octet >= 0; octet--) { ++ if (a.s6_addr[octet] < 255) { ++ a.s6_addr[octet]++; ++ break; ++ } else { ++ a.s6_addr[octet] = 0; ++ } ++ } + -+ return list; ++ return a; +} + -+static void loc_as_list_free(struct loc_as_list* list) { -+ DEBUG(list->ctx, "Releasing AS list at %p\n", list); + LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network, + struct in6_addr* address, unsigned int prefix) { + // Address cannot be unspecified +@@ -160,9 +176,11 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network + LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network, + const char* address_string) { + struct in6_addr first_address; +- unsigned int prefix = 0; + char* prefix_string; +- int r = 1; ++ unsigned int prefix = 128; ++ int r = -EINVAL; + -+ loc_as_list_clear(list); ++ DEBUG(ctx, "Attempting to parse network %s\n", address_string); + + // Make a copy of the string to work on it + char* buffer = strdup(address_string); +@@ -171,29 +189,40 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo + // Split address and prefix + address_string = strsep(&prefix_string, "/"); + +- // Did we find a prefix? ++ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string); + -+ loc_unref(list->ctx); -+ free(list); ++ // Parse the address ++ r = loc_parse_address(ctx, address_string, &first_address); ++ if (r) { ++ DEBUG(ctx, "The address could not be parsed\n"); ++ goto FAIL; ++ } ++ ++ // If a prefix was given, we will try to parse it + if (prefix_string) { + // Convert prefix to integer + prefix = strtol(prefix_string, NULL, 10); + +- if (prefix) { +- // Parse the address +- r = loc_parse_address(ctx, address_string, &first_address); +- +- // Map the prefix to IPv6 if needed +- if (IN6_IS_ADDR_V4MAPPED(&first_address)) +- prefix += 96; ++ if (!prefix) { ++ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string); ++ goto FAIL; + } ++ ++ // Map the prefix to IPv6 if needed ++ if (IN6_IS_ADDR_V4MAPPED(&first_address)) ++ prefix += 96; + } + ++FAIL: + // Free temporary buffer + free(buffer); + +- if (r == 0) { +- r = loc_network_new(ctx, network, &first_address, prefix); +- } ++ // Exit if the parsing was unsuccessful ++ if (r) ++ return r; + +- return r; ++ // Create a new network ++ return loc_network_new(ctx, network, &first_address, prefix); + } + + LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) { +@@ -281,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) { + return network->family; + } + ++LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) { ++ switch (network->family) { ++ case AF_INET6: ++ return network->prefix; ++ ++ case AF_INET: ++ return network->prefix - 96; ++ } ++ ++ return 0; +} + -+LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) { -+ if (!list) -+ return NULL; + static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) { + const size_t length = INET6_ADDRSTRLEN; + +@@ -314,10 +355,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc + return string; + } + ++LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) { ++ return &network->first_address; ++} + -+ if (--list->refcount > 0) -+ return list; + LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) { + return loc_network_format_address(network, &network->first_address); + } + ++LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) { ++ return &network->last_address; ++} ++ + LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) { + return loc_network_format_address(network, &network->last_address); + } +@@ -325,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) { + LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) { + // Address must be larger than the start address + if (in6_addr_cmp(&network->first_address, address) > 0) +- return 1; ++ return 0; + + // Address must be smaller than the last address + if (in6_addr_cmp(&network->last_address, address) < 0) +- return 1; ++ return 0; + + // The address is inside this network +- return 0; ++ return 1; + } + + LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) { +@@ -392,20 +441,310 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag + return loc_network_has_flag(network, flag); + } + +-LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { ++LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) { ++ // Compare address ++ int r = in6_addr_cmp(&self->first_address, &other->first_address); ++ if (r) ++ return r; ++ ++ // Compare prefix ++ if (self->prefix > other->prefix) ++ return 1; ++ else if (self->prefix < other->prefix) ++ return -1; + -+ loc_as_list_free(list); -+ return NULL; ++ // Both networks are equal ++ return 0; +} + -+LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) { -+ return list->size; -+} ++LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { ++ // Either of the start addresses must be in the other subnet ++ if (loc_network_match_address(self, &other->first_address)) ++ return 1; + -+LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { -+ return list->size == 0; -+} ++ if (loc_network_match_address(other, &self->first_address)) ++ return 1; + -+LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_as_unref(list->list[i]); -+} ++ // Or either of the end addresses is in the other subnet ++ if (loc_network_match_address(self, &other->last_address)) ++ return 1; + -+LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { -+ // Check index -+ if (index >= list->size) -+ return NULL; ++ if (loc_network_match_address(other, &self->last_address)) ++ return 1; + -+ return loc_as_ref(list->list[index]); ++ return 0; +} + -+LOC_EXPORT int loc_as_list_append( -+ struct loc_as_list* list, struct loc_as* as) { -+ if (loc_as_list_contains(list, as)) ++LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { ++ // The prefix must be smaller (this avoids the more complex comparisons later) ++ if (self->prefix > other->prefix) + return 0; + -+ // Check if we have space left -+ if (list->size == list->max_size) { -+ ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list); -+ return -ENOMEM; -+ } + // If the start address of the other network is smaller than this network, + // it cannot be a subnet. +- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) ++ if (in6_addr_cmp(&self->first_address, &other->first_address) > 0) + return 0; + + // If the end address of the other network is greater than this network, + // it cannot be a subnet. +- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) ++ if (in6_addr_cmp(&self->last_address, &other->last_address) < 0) + return 0; + + return 1; + } + ++LOC_EXPORT int loc_network_subnets(struct loc_network* network, ++ struct loc_network** subnet1, struct loc_network** subnet2) { ++ int r; ++ *subnet1 = NULL; ++ *subnet2 = NULL; + -+ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); ++ // New prefix length ++ unsigned int prefix = network->prefix + 1; ++ ++ // Check if the new prefix is valid ++ if (valid_prefix(&network->first_address, prefix)) ++ return -1; + -+ list->list[list->size++] = loc_as_ref(as); ++ // Create the first half of the network ++ r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix); ++ if (r) ++ return r; + -+ return 0; -+} ++ // The next subnet starts after the first one ++ struct in6_addr first_address = address_increment(&(*subnet1)->last_address); + -+LOC_EXPORT int loc_as_list_contains( -+ struct loc_as_list* list, struct loc_as* as) { -+ for (unsigned int i = 0; i < list->size; i++) { -+ if (loc_as_cmp(as, list->list[i]) == 0) -+ return 1; ++ // Create the second half of the network ++ r = loc_network_new(network->ctx, subnet2, &first_address, prefix); ++ if (r) ++ return r; ++ ++ // Copy country code ++ const char* country_code = loc_network_get_country_code(network); ++ if (country_code) { ++ loc_network_set_country_code(*subnet1, country_code); ++ loc_network_set_country_code(*subnet2, country_code); ++ } ++ ++ // Copy ASN ++ uint32_t asn = loc_network_get_asn(network); ++ if (asn) { ++ loc_network_set_asn(*subnet1, asn); ++ loc_network_set_asn(*subnet2, asn); + } + ++ // Copy flags ++ loc_network_set_flag(*subnet1, network->flags); ++ loc_network_set_flag(*subnet2, network->flags); ++ + return 0; +} + -+LOC_EXPORT int loc_as_list_contains_number( -+ struct loc_as_list* list, uint32_t number) { -+ struct loc_as* as; ++static int __loc_network_exclude(struct loc_network* network, ++ struct loc_network* other, struct loc_network_list* list) { ++ struct loc_network* subnet1 = NULL; ++ struct loc_network* subnet2 = NULL; + -+ int r = loc_as_new(list->ctx, &as, number); ++ int r = loc_network_subnets(network, &subnet1, &subnet2); + if (r) -+ return -1; ++ goto ERROR; + -+ r = loc_as_list_contains(list, as); -+ loc_as_unref(as); ++ if (loc_network_cmp(other, subnet1) == 0) { ++ r = loc_network_list_push(list, subnet2); ++ if (r) ++ goto ERROR; + -+ return r; -+} -diff --git a/src/database.c b/src/database.c -index 29823b2..51cb5cd 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -38,6 +38,7 @@ - - #include - #include -+#include - #include - #include - #include -@@ -101,7 +102,7 @@ struct loc_database_enumerator { - // Search string - char* string; - struct loc_country_list* countries; -- uint32_t asn; -+ struct loc_as_list* asns; - enum loc_network_flags flags; - int family; - -@@ -1036,9 +1037,20 @@ LOC_EXPORT int loc_database_enumerator_set_countries( - return 0; - } - --LOC_EXPORT int loc_database_enumerator_set_asn( -- struct loc_database_enumerator* enumerator, unsigned int asn) { -- enumerator->asn = asn; -+LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns( -+ struct loc_database_enumerator* enumerator) { -+ if (!enumerator->asns) -+ return NULL; ++ } else if (loc_network_cmp(other, subnet2) == 0) { ++ r = loc_network_list_push(list, subnet1); ++ if (r) ++ goto ERROR; + -+ return loc_as_list_ref(enumerator->asns); -+} ++ } else if (loc_network_is_subnet(subnet1, other)) { ++ r = loc_network_list_push(list, subnet2); ++ if (r) ++ goto ERROR; + -+LOC_EXPORT int loc_database_enumerator_set_asns( -+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) { -+ if (enumerator->asns) -+ loc_as_list_unref(enumerator->asns); ++ r = __loc_network_exclude(subnet1, other, list); ++ if (r) ++ goto ERROR; + -+ enumerator->asns = loc_as_list_ref(asns); - - return 0; - } -@@ -1123,6 +1135,12 @@ static int loc_network_match_countries(struct loc_network* network, struct loc_c - return loc_country_list_contains_code(countries, country_code); - } - -+static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { -+ uint32_t asn = loc_network_get_asn(network); ++ } else if (loc_network_is_subnet(subnet2, other)) { ++ r = loc_network_list_push(list, subnet1); ++ if (r) ++ goto ERROR; + -+ return loc_as_list_contains_number(asns, asn); -+} ++ r = __loc_network_exclude(subnet2, other, list); ++ if (r) ++ goto ERROR; + - static int loc_database_enumerator_filter_network( - struct loc_database_enumerator* enumerator, struct loc_network* network) { - // Skip if the family does not match -@@ -1134,8 +1152,7 @@ static int loc_database_enumerator_filter_network( - return 1; - - // Skip if the ASN does not match -- if (enumerator->asn && -- !loc_network_match_asn(network, enumerator->asn)) -+ if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) - return 1; - - // Skip if flags do not match -diff --git a/src/libloc.sym b/src/libloc.sym -index 40e9f88..53273cd 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -37,6 +37,18 @@ global: - loc_as_set_name; - loc_as_unref; - -+ # AS List -+ loc_as_list_append; -+ loc_as_list_clear; -+ loc_as_list_contains; -+ loc_as_list_contains_number; -+ loc_as_list_empty; -+ loc_as_list_get; -+ loc_as_list_new; -+ loc_as_list_ref; -+ loc_as_list_size; -+ loc_as_list_unref; ++ } else { ++ ERROR(network->ctx, "We should never get here\n"); ++ r = 1; ++ goto ERROR; ++ } + - # Country - loc_country_cmp; - loc_country_code_is_valid; -@@ -78,13 +90,14 @@ global: - loc_database_verify; - - # Database Enumerator -+ loc_database_enumerator_get_asns; - loc_database_enumerator_get_countries; - loc_database_enumerator_new; - loc_database_enumerator_next_as; - loc_database_enumerator_next_country; - loc_database_enumerator_next_network; - loc_database_enumerator_ref; -- loc_database_enumerator_set_asn; -+ loc_database_enumerator_set_asns; - loc_database_enumerator_set_countries; - loc_database_enumerator_set_family; - loc_database_enumerator_set_flag; -diff --git a/src/loc/as-list.h b/src/loc/as-list.h -new file mode 100644 -index 0000000..7b5c4e8 ---- /dev/null -+++ b/src/loc/as-list.h -@@ -0,0 +1,41 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet ++ERROR: ++ if (subnet1) ++ loc_network_unref(subnet1); + -+ Copyright (C) 2017 IPFire Development Team ++ if (subnet2) ++ loc_network_unref(subnet2); + -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. ++ return r; ++} + -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ ++static int __loc_network_exclude_to_list(struct loc_network* self, ++ struct loc_network* other, struct loc_network_list* list) { ++ // Other must be a subnet of self ++ if (!loc_network_is_subnet(self, other)) { ++ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); ++ ++ // Exit silently ++ return 0; ++ } ++ ++ // We cannot perform this operation if both networks equal ++ if (loc_network_cmp(self, other) == 0) { ++ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); ++ ++ // Exit silently ++ return 0; ++ } + -+#ifndef LIBLOC_AS_LIST_H -+#define LIBLOC_AS_LIST_H ++ return __loc_network_exclude(self, other, list); ++} + -+#include -+#include ++LOC_EXPORT struct loc_network_list* loc_network_exclude( ++ struct loc_network* self, struct loc_network* other) { ++ struct loc_network_list* list; + -+struct loc_as_list; ++#ifdef ENABLE_DEBUG ++ char* n1 = loc_network_str(self); ++ char* n2 = loc_network_str(other); + -+int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list); -+struct loc_as_list* loc_as_list_ref(struct loc_as_list* list); -+struct loc_as_list* loc_as_list_unref(struct loc_as_list* list); ++ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); + -+size_t loc_as_list_size(struct loc_as_list* list); -+int loc_as_list_empty(struct loc_as_list* list); -+void loc_as_list_clear(struct loc_as_list* list); ++ free(n1); ++ free(n2); ++#endif + -+struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index); -+int loc_as_list_append(struct loc_as_list* list, struct loc_as* as); ++ // Create a new list with the result ++ int r = loc_network_list_new(self->ctx, &list); ++ if (r) { ++ ERROR(self->ctx, "Could not create network list: %d\n", r); + -+int loc_as_list_contains( -+ struct loc_as_list* list, struct loc_as* as); -+int loc_as_list_contains_number( -+ struct loc_as_list* list, uint32_t number); ++ return NULL; ++ } + -+#endif -diff --git a/src/loc/database.h b/src/loc/database.h -index 246e5c5..70801f0 100644 ---- a/src/loc/database.h -+++ b/src/loc/database.h -@@ -70,7 +70,10 @@ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerato - struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); - int loc_database_enumerator_set_countries( - struct loc_database_enumerator* enumerator, struct loc_country_list* countries); --int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn); -+struct loc_as_list* loc_database_enumerator_get_asns( -+ struct loc_database_enumerator* enumerator); -+int loc_database_enumerator_set_asns( -+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns); - int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); - int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); - int loc_database_enumerator_next_as( -diff --git a/src/python/database.c b/src/python/database.c -index e6f6f37..38a804c 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -17,6 +17,8 @@ - #include - - #include -+#include -+#include - #include - - #include "locationmodule.h" -@@ -258,15 +260,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { - } - - static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { -- char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL }; -+ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL }; - PyObject* country_codes = NULL; -- unsigned int asn = 0; -+ PyObject* asn_list = NULL; - int flags = 0; - int family = 0; - int flatten = 0; - -- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist, -- &PyList_Type, &country_codes, &asn, &flags, &family, &flatten)) -+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist, -+ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten)) - return NULL; - - struct loc_database_enumerator* enumerator; -@@ -330,13 +332,57 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - } - - // Set the ASN we are searching for -- if (asn) { -- r = loc_database_enumerator_set_asn(enumerator, asn); -+ if (asn_list) { -+ struct loc_as_list* asns; -+ r = loc_as_list_new(loc_ctx, &asns); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not create AS list"); -+ return NULL; -+ } ++ r = __loc_network_exclude_to_list(self, other, list); ++ if (r) { ++ loc_network_list_unref(list); + -+ for (unsigned int i = 0; i < PyList_Size(asn_list); i++) { -+ PyObject* item = PyList_GetItem(asn_list, i); ++ return NULL; ++ } + -+ if (!PyLong_Check(item)) { -+ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers"); ++ // Return the result ++ return list; ++} + -+ loc_as_list_unref(asns); -+ return NULL; -+ } ++LOC_EXPORT struct loc_network_list* loc_network_exclude_list( ++ struct loc_network* network, struct loc_network_list* list) { ++ struct loc_network_list* to_check; + -+ unsigned long number = PyLong_AsLong(item); - -+ struct loc_as* as; -+ r = loc_as_new(loc_ctx, &as, number); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not create AS"); ++ // Create a new list with all networks to look at ++ int r = loc_network_list_new(network->ctx, &to_check); ++ if (r) ++ return NULL; + -+ loc_as_list_unref(asns); -+ loc_as_unref(as); -+ return NULL; -+ } ++ struct loc_network* subnet = NULL; ++ struct loc_network_list* subnets = NULL; + -+ r = loc_as_list_append(asns, as); ++ for (unsigned int i = 0; i < loc_network_list_size(list); i++) { ++ subnet = loc_network_list_get(list, i); ++ ++ // Find all excluded networks ++ if (!loc_network_list_contains(to_check, subnet)) { ++ r = __loc_network_exclude_to_list(network, subnet, to_check); + if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list"); ++ loc_network_list_unref(to_check); ++ loc_network_unref(subnet); + -+ loc_as_list_unref(asns); -+ loc_as_unref(as); + return NULL; + } -+ -+ loc_as_unref(as); + } + -+ r = loc_database_enumerator_set_asns(enumerator, asns); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); -+ -+ loc_as_list_unref(asns); - return NULL; - } -+ -+ loc_as_list_unref(asns); - } - - // Set the flags we are searching for -diff --git a/src/python/export.py b/src/python/export.py -index 739742f..f675eb3 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -197,7 +197,7 @@ class Exporter(object): - - # Get all networks that match the family - networks = self.db.search_networks(family=family, -- country_codes=country_codes, flags=flags, flatten=True) -+ country_codes=country_codes, asns=asns, flags=flags, flatten=True) - - # Walk through all networks - for network in networks: --- -2.20.1 - -From 50120b991fc2fa4b7813096de87b42d700faf3e6 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 17 Nov 2020 16:56:43 +0000 -Subject: [PATCH 060/111] database: Simplify network matching code - -Signed-off-by: Michael Tremer ---- - src/database.c | 16 ++++++++-------- - 1 file changed, 8 insertions(+), 8 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 51cb5cd..1a354f6 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } - --static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) { -- const char* country_code = loc_network_get_country_code(network); -- -- return loc_country_list_contains_code(countries, country_code); --} -- - static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { - uint32_t asn = loc_network_get_asn(network); - -@@ -1148,8 +1142,14 @@ static int loc_database_enumerator_filter_network( - return 1; - - // Skip if the country code does not match -- if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries)) -- return 1; -+ if (enumerator->countries) { -+ if (!loc_country_list_empty(enumerator->countries)) { -+ const char* country_code = loc_network_get_country_code(network); -+ -+ if (!loc_country_list_contains_code(enumerator->countries, country_code)) -+ return 1; -+ } ++ // Cleanup ++ loc_network_unref(subnet); + } - - // Skip if the ASN does not match - if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) --- -2.20.1 - -From c1a36c943181da5cd2aef589a972d5027e529eb8 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 17 Nov 2020 16:58:55 +0000 -Subject: [PATCH 061/111] database: Simplify AS matching code - -Signed-off-by: Michael Tremer ---- - src/database.c | 16 ++++++++-------- - 1 file changed, 8 insertions(+), 8 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 1a354f6..be93e00 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } - --static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { -- uint32_t asn = loc_network_get_asn(network); -- -- return loc_as_list_contains_number(asns, asn); --} -- - static int loc_database_enumerator_filter_network( - struct loc_database_enumerator* enumerator, struct loc_network* network) { - // Skip if the family does not match -@@ -1152,8 +1146,14 @@ static int loc_database_enumerator_filter_network( - } - - // Skip if the ASN does not match -- if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) -- return 1; -+ if (enumerator->asns) { -+ if (!loc_as_list_empty(enumerator->asns)) { -+ uint32_t asn = loc_network_get_asn(network); + -+ if (!loc_as_list_contains_number(enumerator->asns, asn)) -+ return 1; -+ } ++ r = loc_network_list_new(network->ctx, &subnets); ++ if (r) { ++ loc_network_list_unref(to_check); ++ return NULL; + } - - // Skip if flags do not match - if (enumerator->flags && --- -2.20.1 - -From d5205091f9cc1ff987e483325d48696459df08d8 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 17 Nov 2020 17:50:17 +0000 -Subject: [PATCH 062/111] countries: Make list grow dynamically - -Signed-off-by: Michael Tremer ---- - src/country-list.c | 38 ++++++++++++++++++++++++++------------ - 1 file changed, 26 insertions(+), 12 deletions(-) - -diff --git a/src/country-list.c b/src/country-list.c -index ae0d71a..1ce2d06 100644 ---- a/src/country-list.c -+++ b/src/country-list.c -@@ -25,11 +25,27 @@ struct loc_country_list { - struct loc_ctx* ctx; - int refcount; - -- struct loc_country* list[1024]; -+ struct loc_country** elements; -+ size_t elements_size; + - size_t size; -- size_t max_size; - }; - -+static int loc_country_list_grow(struct loc_country_list* list, size_t size) { -+ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); ++ off_t smallest_subnet = 0; + -+ struct loc_country** elements = reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; ++ while (!loc_network_list_empty(to_check)) { ++ struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check); + -+ list->elements = elements; -+ list->elements_size += size; ++ // Check whether the subnet to check is part of the input list ++ if (loc_network_list_contains(list, subnet_to_check)) { ++ loc_network_unref(subnet_to_check); ++ continue; ++ } + -+ return 0; -+} ++ // Marks whether this subnet passed all checks ++ int passed = 1; + - LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, - struct loc_country_list** list) { - struct loc_country_list* l = calloc(1, sizeof(*l)); -@@ -39,9 +55,6 @@ LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, - l->ctx = loc_ref(ctx); - l->refcount = 1; - -- // Do not allow this list to grow larger than this -- l->max_size = 1024; -- - DEBUG(l->ctx, "Country list allocated at %p\n", l); - *list = l; - -@@ -84,7 +97,7 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { - - LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { - for (unsigned int i = 0; i < list->size; i++) -- loc_country_unref(list->list[i]); -+ loc_country_unref(list->elements[i]); - } - - LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { -@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* lis - if (index >= list->size) - return NULL; - -- return loc_country_ref(list->list[index]); -+ return loc_country_ref(list->elements[index]); - } - - LOC_EXPORT int loc_country_list_append( -@@ -101,14 +114,15 @@ LOC_EXPORT int loc_country_list_append( - return 0; - - // Check if we have space left -- if (list->size == list->max_size) { -- ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list); -- return -ENOMEM; -+ if (list->size >= list->elements_size) { -+ int r = loc_country_list_grow(list, 64); -+ if (r) -+ return r; - } - - DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); - -- list->list[list->size++] = loc_country_ref(country); -+ list->elements[list->size++] = loc_country_ref(country); - - return 0; - } -@@ -116,7 +130,7 @@ LOC_EXPORT int loc_country_list_append( - LOC_EXPORT int loc_country_list_contains( - struct loc_country_list* list, struct loc_country* country) { - for (unsigned int i = 0; i < list->size; i++) { -- if (loc_country_cmp(country, list->list[i]) == 0) -+ if (loc_country_cmp(country, list->elements[i]) == 0) - return 1; - } - --- -2.20.1 - -From 3b44e4211371d2103f89ba8f056b15edb7778fac Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 17 Nov 2020 17:55:51 +0000 -Subject: [PATCH 063/111] networks: Make list grow dynamically - -Signed-off-by: Michael Tremer ---- - src/network-list.c | 60 ++++++++++++++++++++++++++++------------------ - 1 file changed, 37 insertions(+), 23 deletions(-) - -diff --git a/src/network-list.c b/src/network-list.c -index 1f6e80e..4912c02 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -25,11 +25,27 @@ struct loc_network_list { - struct loc_ctx* ctx; - int refcount; - -- struct loc_network* list[1024]; -+ struct loc_network** elements; -+ size_t elements_size; ++ for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) { ++ subnet = loc_network_list_get(list, i); ++ ++ // Drop this subnet if is a subnet of another subnet ++ if (loc_network_is_subnet(subnet, subnet_to_check)) { ++ passed = 0; ++ loc_network_unref(subnet); ++ break; ++ } + - size_t size; -- size_t max_size; - }; - -+static int loc_network_list_grow(struct loc_network_list* list, size_t size) { -+ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); ++ // Break it down if it overlaps ++ if (loc_network_overlaps(subnet, subnet_to_check)) { ++ passed = 0; + -+ struct loc_network** elements = reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; ++ __loc_network_exclude_to_list(subnet_to_check, subnet, to_check); + -+ list->elements = elements; -+ list->elements_size += size; ++ loc_network_unref(subnet); ++ break; ++ } + -+ return 0; -+} ++ // If the subnet is strictly greater, we do not need to continue the search ++ r = loc_network_cmp(subnet, subnet_to_check); ++ if (r > 0) { ++ loc_network_unref(subnet); ++ break; + - LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, - struct loc_network_list** list) { - struct loc_network_list* l = calloc(1, sizeof(*l)); -@@ -39,9 +55,6 @@ LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, - l->ctx = loc_ref(ctx); - l->refcount = 1; - -- // Do not allow this list to grow larger than this -- l->max_size = 1024; -- - DEBUG(l->ctx, "Network list allocated at %p\n", l); - *list = l; - return 0; -@@ -57,7 +70,7 @@ static void loc_network_list_free(struct loc_network_list* list) { - DEBUG(list->ctx, "Releasing network list at %p\n", list); - - for (unsigned int i = 0; i < list->size; i++) -- loc_network_unref(list->list[i]); -+ loc_network_unref(list->elements[i]); - - loc_unref(list->ctx); - free(list); -@@ -84,7 +97,7 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { - - LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { - for (unsigned int i = 0; i < list->size; i++) -- loc_network_unref(list->list[i]); -+ loc_network_unref(list->elements[i]); - - list->size = 0; - } -@@ -94,7 +107,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { - char* s; - - for (unsigned int i = 0; i < list->size; i++) { -- network = list->list[i]; -+ network = list->elements[i]; - - s = loc_network_str(network); - -@@ -108,7 +121,7 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis - if (index >= list->size) - return NULL; - -- return loc_network_ref(list->list[index]); -+ return loc_network_ref(list->elements[index]); - } - - LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -@@ -117,14 +130,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n - return 0; - - // Check if we have space left -- if (list->size == list->max_size) { -- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); -- return -ENOMEM; -+ if (list->size >= list->elements_size) { -+ int r = loc_network_list_grow(list, 64); -+ if (r) -+ return r; - } - - DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); - -- list->list[list->size++] = loc_network_ref(network); -+ list->elements[list->size++] = loc_network_ref(network); - - return 0; - } -@@ -136,7 +150,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis - return NULL; - } - -- struct loc_network* network = list->list[--list->size]; -+ struct loc_network* network = list->elements[--list->size]; - - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); - -@@ -150,11 +164,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis - return NULL; - } - -- struct loc_network* network = list->list[0]; -+ struct loc_network* network = list->elements[0]; - - // Move all elements to the top of the stack - for (unsigned int i = 0; i < --list->size; i++) { -- list->list[i] = list->list[i+1]; -+ list->elements[i] = list->elements[i+1]; - } - - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -@@ -164,7 +178,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis - - LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { - for (unsigned int i = 0; i < list->size; i++) { -- if (loc_network_eq(list->list[i], network)) -+ if (loc_network_eq(list->elements[i], network)) - return 1; - } - -@@ -176,11 +190,11 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1 - if (i1 >= list->size || i2 >= list->size) - return; - -- struct loc_network* network1 = list->list[i1]; -- struct loc_network* network2 = list->list[i2]; -+ struct loc_network* network1 = list->elements[i1]; -+ struct loc_network* network2 = list->elements[i2]; - -- list->list[i1] = network2; -- list->list[i2] = network1; -+ list->elements[i1] = network2; -+ list->elements[i2] = network1; - } - - LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { -@@ -200,7 +214,7 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { - swapped = 0; - - for (unsigned int i = 1; i < n; i++) { -- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { -+ if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) { - loc_network_list_swap(list, i-1, i); - swapped = 1; - } -@@ -215,7 +229,7 @@ LOC_EXPORT int loc_network_list_merge( - int r; - - for (unsigned int i = 0; i < other->size; i++) { -- r = loc_network_list_push(self, other->list[i]); -+ r = loc_network_list_push(self, other->elements[i]); - if (r) - return r; - } --- -2.20.1 - -From 1a415f8c555f4fe9a68eb2a897c4a1fc0d33db25 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 17 Nov 2020 17:57:55 +0000 -Subject: [PATCH 064/111] as: Make lists grow dynamically - -Signed-off-by: Michael Tremer ---- - src/as-list.c | 38 ++++++++++++++++++++++++++------------ - 1 file changed, 26 insertions(+), 12 deletions(-) - -diff --git a/src/as-list.c b/src/as-list.c -index 7c69eb0..17de23e 100644 ---- a/src/as-list.c -+++ b/src/as-list.c -@@ -25,11 +25,27 @@ struct loc_as_list { - struct loc_ctx* ctx; - int refcount; - -- struct loc_as* list[1024]; -+ struct loc_as** elements; -+ size_t elements_size; ++ // If it is strictly smaller, we can continue the search from here next ++ // time because all networks that are to be checked can only be larger ++ // than this one. ++ } else if (r < 0) { ++ smallest_subnet = i; ++ } + - size_t size; -- size_t max_size; - }; - -+static int loc_as_list_grow(struct loc_as_list* list, size_t size) { -+ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); ++ loc_network_unref(subnet); ++ } + -+ struct loc_as** elements = reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; ++ if (passed) { ++ r = loc_network_list_push(subnets, subnet_to_check); ++ } + -+ list->elements = elements; -+ list->elements_size += size; ++ loc_network_unref(subnet_to_check); ++ } + -+ return 0; ++ loc_network_list_unref(to_check); ++ ++ return subnets; +} + - LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, - struct loc_as_list** list) { - struct loc_as_list* l = calloc(1, sizeof(*l)); -@@ -39,9 +55,6 @@ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, - l->ctx = loc_ref(ctx); - l->refcount = 1; - -- // Do not allow this list to grow larger than this -- l->max_size = 1024; -- - DEBUG(l->ctx, "AS list allocated at %p\n", l); - *list = l; - -@@ -84,7 +97,7 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { - - LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { - for (unsigned int i = 0; i < list->size; i++) -- loc_as_unref(list->list[i]); -+ loc_as_unref(list->elements[i]); - } - - LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { -@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index - if (index >= list->size) - return NULL; - -- return loc_as_ref(list->list[index]); -+ return loc_as_ref(list->elements[index]); - } - - LOC_EXPORT int loc_as_list_append( -@@ -101,14 +114,15 @@ LOC_EXPORT int loc_as_list_append( - return 0; - - // Check if we have space left -- if (list->size == list->max_size) { -- ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list); -- return -ENOMEM; -+ if (list->size >= list->elements_size) { -+ int r = loc_as_list_grow(list, 64); -+ if (r) -+ return r; - } - - DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); - -- list->list[list->size++] = loc_as_ref(as); -+ list->elements[list->size++] = loc_as_ref(as); + LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { + // Add country code + loc_country_code_copy(dbobj->country_code, network->country_code); +@@ -474,7 +813,7 @@ struct loc_network_tree_node { + struct loc_network* network; + }; +-LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { ++int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { + struct loc_network_tree* t = calloc(1, sizeof(*t)); + if (!t) + return -ENOMEM; +@@ -494,7 +833,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree return 0; } -@@ -116,7 +130,7 @@ LOC_EXPORT int loc_as_list_append( - LOC_EXPORT int loc_as_list_contains( - struct loc_as_list* list, struct loc_as* as) { - for (unsigned int i = 0; i < list->size; i++) { -- if (loc_as_cmp(as, list->list[i]) == 0) -+ if (loc_as_cmp(as, list->elements[i]) == 0) - return 1; - } - --- -2.20.1 - -From e6592434ee7836507c1f436ec3b0db3bc81a81b9 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 17 Nov 2020 18:13:49 +0000 -Subject: [PATCH 065/111] export: Change back to use Network objects - -Signed-off-by: Michael Tremer ---- - src/python/export.py | 4 +--- - 1 file changed, 1 insertion(+), 3 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index f675eb3..67e437f 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -144,9 +144,7 @@ class XTGeoIPOutputWriter(OutputWriter): - def _write_network(self, network): - for address in (network.first_address, network.last_address): - # Convert this into a string of bits -- bytes = socket.inet_pton( -- socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, -- ) -+ bytes = socket.inet_pton(network.family, address) - - self.f.write(bytes) - --- -2.20.1 - -From 248f5e0419f2349253b8ea96e477c15649fe2173 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 17 Nov 2020 18:14:15 +0000 -Subject: [PATCH 066/111] Actually clear all lists - -Signed-off-by: Michael Tremer ---- - src/as-list.c | 8 ++++++++ - src/country-list.c | 8 ++++++++ - src/network-list.c | 6 ++++++ - 3 files changed, 22 insertions(+) - -diff --git a/src/as-list.c b/src/as-list.c -index 17de23e..76620c7 100644 ---- a/src/as-list.c -+++ b/src/as-list.c -@@ -96,8 +96,16 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { - } - - LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { -+ if (!list->elements) -+ return; -+ - for (unsigned int i = 0; i < list->size; i++) - loc_as_unref(list->elements[i]); -+ -+ free(list->elements); -+ list->elements_size = 0; -+ -+ list->size = 0; - } - LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { -diff --git a/src/country-list.c b/src/country-list.c -index 1ce2d06..1c49c47 100644 ---- a/src/country-list.c -+++ b/src/country-list.c -@@ -96,8 +96,16 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { ++struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { + return loc_network_tree_node_ref(tree->root); } - LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { -+ if (!list->elements) -+ return; -+ - for (unsigned int i = 0; i < list->size; i++) - loc_country_unref(list->elements[i]); -+ -+ free(list->elements); -+ list->elements_size = 0; -+ -+ list->size = 0; +@@ -566,7 +905,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_ + return 0; } - LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { -diff --git a/src/network-list.c b/src/network-list.c -index 4912c02..9cb4547 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -96,9 +96,15 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { +-LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree, ++int loc_network_tree_walk(struct loc_network_tree* tree, + int(*filter_callback)(struct loc_network* network, void* data), + int(*callback)(struct loc_network* network, void* data), void* data) { + return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data); +@@ -581,7 +920,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) { + free(tree); } - LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { -+ if (!list->elements) -+ return; -+ - for (unsigned int i = 0; i < list->size; i++) - loc_network_unref(list->elements[i]); +-LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { ++struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { + if (--tree->refcount > 0) + return tree; -+ free(list->elements); -+ list->elements_size = 0; -+ - list->size = 0; +@@ -602,13 +941,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) { + return 0; } --- -2.20.1 - -From c98ebf8aae2aa141193db52cd9429b1ded5b09c4 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 17 Nov 2020 18:34:51 +0000 -Subject: [PATCH 067/111] database: Do not clean up python list - -Signed-off-by: Michael Tremer ---- - src/python/database.c | 9 +++++++-- - 1 file changed, 7 insertions(+), 2 deletions(-) - -diff --git a/src/python/database.c b/src/python/database.c -index 38a804c..ed22275 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -325,9 +325,14 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - loc_country_unref(country); - } - -- loc_database_enumerator_set_countries(enumerator, countries); -+ r = loc_database_enumerator_set_countries(enumerator, countries); -+ if (r) { -+ PyErr_SetFromErrno(PyExc_SystemError); -+ -+ loc_as_list_unref(countries); -+ return NULL; -+ } - -- Py_DECREF(country_codes); - loc_country_list_unref(countries); - } - --- -2.20.1 - -From 5470d06cb59027f4e04b6d576763dbf7f1093fde Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 17 Nov 2020 19:01:04 +0000 -Subject: [PATCH 068/111] database: Free filter lists in enumerator - -Signed-off-by: Michael Tremer ---- - src/database.c | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/src/database.c b/src/database.c -index be93e00..ca35fe1 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -950,6 +950,12 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera - if (enumerator->string) - free(enumerator->string); - -+ if (enumerator->countries) -+ loc_country_list_unref(enumerator->countries); -+ -+ if (enumerator->asns) -+ loc_as_list_unref(enumerator->asns); -+ - // Free network search - free(enumerator->networks_visited); - --- -2.20.1 - -From e0e96878d3df51c4a265d51d088005dedf9335e3 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 18 Nov 2020 13:18:52 +0000 -Subject: [PATCH 069/111] database: Add debug output to filtering - -Signed-off-by: Michael Tremer ---- - src/database.c | 29 ++++++++++++++++------------- - 1 file changed, 16 insertions(+), 13 deletions(-) - -diff --git a/src/database.c b/src/database.c -index ca35fe1..83dd752 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1138,33 +1138,36 @@ static int loc_database_enumerator_stack_push_node( - static int loc_database_enumerator_filter_network( - struct loc_database_enumerator* enumerator, struct loc_network* network) { - // Skip if the family does not match -- if (enumerator->family && loc_network_address_family(network) != enumerator->family) -+ if (enumerator->family && loc_network_address_family(network) != enumerator->family) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network); - return 1; -+ } - - // Skip if the country code does not match -- if (enumerator->countries) { -- if (!loc_country_list_empty(enumerator->countries)) { -- const char* country_code = loc_network_get_country_code(network); -+ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) { -+ const char* country_code = loc_network_get_country_code(network); - -- if (!loc_country_list_contains_code(enumerator->countries, country_code)) -- return 1; -+ if (!loc_country_list_contains_code(enumerator->countries, country_code)) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network); -+ return 1; - } - } - - // Skip if the ASN does not match -- if (enumerator->asns) { -- if (!loc_as_list_empty(enumerator->asns)) { -- uint32_t asn = loc_network_get_asn(network); -+ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) { -+ uint32_t asn = loc_network_get_asn(network); - -- if (!loc_as_list_contains_number(enumerator->asns, asn)) -- return 1; -+ if (!loc_as_list_contains_number(enumerator->asns, asn)) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network); -+ return 1; - } - } +-LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) { ++int loc_network_tree_dump(struct loc_network_tree* tree) { + DEBUG(tree->ctx, "Dumping network tree at %p\n", tree); - // Skip if flags do not match -- if (enumerator->flags && -- !loc_network_match_flag(network, enumerator->flags)) -+ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network); - return 1; -+ } + return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL); + } + +-LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { ++int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { + DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree); - // Do not filter + struct loc_network_tree_node* node = loc_network_tree_get_path(tree, +@@ -639,7 +978,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) { return 0; --- -2.20.1 - -From bce0c9295ff8ff9488f24babe01ce851228d0b1e Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 18 Nov 2020 13:19:04 +0000 -Subject: [PATCH 070/111] export: Remove filtering for flags - -The filter is an AND filter and if we set the flags from -the special country codes, we won't get back much. - -Signed-off-by: Michael Tremer ---- - src/python/export.py | 8 +------- - 1 file changed, 1 insertion(+), 7 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index 67e437f..4219957 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -187,15 +187,9 @@ class Exporter(object): - country_code for country_code in countries if not country_code in FLAGS.values() - ] - -- # Collect flags -- flags = 0 -- for flag in FLAGS: -- if FLAGS[flag] in countries: -- flags |= flag -- - # Get all networks that match the family - networks = self.db.search_networks(family=family, -- country_codes=country_codes, asns=asns, flags=flags, flatten=True) -+ country_codes=country_codes, asns=asns, flatten=True) + } - # Walk through all networks - for network in networks: --- -2.20.1 - -From 627bf1daaae1510cfd4016297ed16b82df209aae Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 18 Nov 2020 13:33:45 +0000 -Subject: [PATCH 071/111] python: Remove unnecessary db object from writers - -Signed-off-by: Michael Tremer ---- - src/python/export.py | 12 ++++++------ - 1 file changed, 6 insertions(+), 6 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index 4219957..5bc9f30 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -39,8 +39,8 @@ class OutputWriter(object): - suffix = "networks" - mode = "w" +-LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { ++size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { + size_t counter = 0; -- def __init__(self, db, f, prefix=None, flatten=True): -- self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten -+ def __init__(self, f, prefix=None, flatten=True): -+ self.f, self.prefix, self.flatten = f, prefix, flatten + int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter); +@@ -661,11 +1000,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node) + return counter; + } - # The previously written network - self._last_network = None -@@ -49,13 +49,13 @@ class OutputWriter(object): - self._write_header() +-LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { ++size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { + return __loc_network_tree_count_nodes(tree->root); + } - @classmethod -- def open(cls, db, filename, **kwargs): -+ def open(cls, filename, **kwargs): - """ - Convenience function to open a file - """ - f = open(filename, cls.mode) +-LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { ++int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { + struct loc_network_tree_node* n = calloc(1, sizeof(*n)); + if (!n) + return -ENOMEM; +@@ -680,7 +1019,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network + return 0; + } -- return cls(db, f, **kwargs) -+ return cls(f, **kwargs) +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { ++struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { + if (node) + node->refcount++; - def __repr__(self): - return "<%s f=%s>" % (self.__class__.__name__, self.f) -@@ -172,7 +172,7 @@ class Exporter(object): - directory, prefix=country_code, suffix=self.writer.suffix, family=family, - ) +@@ -703,7 +1042,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) { + free(node); + } -- writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code) -+ writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code) +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { ++struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { + if (!node) + return NULL; - # Create writers for ASNs - for asn in asns: -@@ -180,7 +180,7 @@ class Exporter(object): - directory, "AS%s" % asn, suffix=self.writer.suffix, family=family, - ) +@@ -714,7 +1053,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_ + return NULL; + } -- writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) -+ writers[asn] = self.writer.open(filename, prefix="AS%s" % asn) - - # Filter countries from special country codes - country_codes = [ --- -2.20.1 - -From 9cb56ac9adafafa6e452009c2fa2d42e94474e11 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 18 Nov 2020 13:34:50 +0000 -Subject: [PATCH 072/111] location: End lookup after an invalid IP address was - passed - -Signed-off-by: Michael Tremer ---- - src/python/location.in | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/python/location.in b/src/python/location.in -index 0d09210..6885ea0 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -253,6 +253,7 @@ class CLI(object): - network = db.lookup(address) - except ValueError: - print(_("Invalid IP address: %s") % address, file=sys.stderr) -+ return 2 +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { ++struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { + if (index == 0) + node = node->zero; + else +@@ -726,10 +1065,10 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne + return loc_network_tree_node_ref(node); + } - args = { - "address" : address, --- -2.20.1 - -From 2a550d12208f8bc8002e05ac08613312df26b20e Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 19 Nov 2020 12:03:33 +0000 -Subject: [PATCH 073/111] python: Fix download of database - -This was all messed up in 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 -when the change of type for the timestamp wasn't changed everywhere -else. - -Signed-off-by: Michael Tremer ---- - src/python/downloader.py | 6 +++--- - src/python/location.in | 2 +- - 2 files changed, 4 insertions(+), 4 deletions(-) - -diff --git a/src/python/downloader.py b/src/python/downloader.py -index 87bbb68..05f7872 100644 ---- a/src/python/downloader.py -+++ b/src/python/downloader.py -@@ -119,8 +119,8 @@ class Downloader(object): +-LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { ++int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { + return (!!node->network); + } - headers = {} - if timestamp: -- headers["If-Modified-Since"] = timestamp.strftime( -- "%a, %d %b %Y %H:%M:%S GMT", -+ headers["If-Modified-Since"] = time.strftime( -+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp), - ) +-LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { ++struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { + return loc_network_ref(node->network); + } +diff --git a/src/perl/Location.xs b/src/perl/Location.xs +index dcf3f0d..b7676d2 100644 +--- a/src/perl/Location.xs ++++ b/src/perl/Location.xs +@@ -125,7 +125,7 @@ database_countries(db) + PPCODE: + // Create Database enumerator + struct loc_database_enumerator* enumerator; +- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES); ++ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0); - t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) -@@ -195,7 +195,7 @@ class Downloader(object): - db = Database(f.name) + if (err) { + croak("Could not create a database enumerator\n"); +diff --git a/src/python/database.c b/src/python/database.c +index 1013a58..f385c61 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -17,6 +17,8 @@ + #include - # Database is not recent -- if timestamp and db.created_at < timestamp.timestamp(): -+ if timestamp and db.created_at < timestamp: - return False + #include ++#include ++#include + #include - log.info("Downloaded new database from %s" % (time.strftime( -diff --git a/src/python/location.in b/src/python/location.in -index 6885ea0..b30beae 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -433,7 +433,7 @@ class CLI(object): + #include "locationmodule.h" +@@ -207,10 +209,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database + return (PyObject*)self; + } - # Try downloading a new database - try: -- t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir) -+ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir) +-static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) { ++static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) { + struct loc_database_enumerator* enumerator; - # If no file could be downloaded, log a message - except FileNotFoundError as e: --- -2.20.1 - -From a1a00053300cff3c0f690d52377c76c83c2a08b2 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 19 Nov 2020 12:34:11 +0000 -Subject: [PATCH 074/111] python: Add property to return IP addresses as bytes - -This avoids calling inet_pton to parse IP addresses from string - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 2 ++ - src/loc/network.h | 2 ++ - src/network.c | 8 ++++++++ - src/python/database.c | 2 +- - src/python/export.py | 7 ++----- - src/python/network.c | 40 ++++++++++++++++++++++++++++++++++++++++ - 6 files changed, 55 insertions(+), 6 deletions(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 53273cd..406dd15 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -113,6 +113,8 @@ global: - loc_network_format_last_address; - loc_network_get_asn; - loc_network_get_country_code; -+ loc_network_get_first_address; -+ loc_network_get_last_address; - loc_network_gt; - loc_network_has_flag; - loc_network_is_subnet; -diff --git a/src/loc/network.h b/src/loc/network.h -index d86b685..4b7410c 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -39,7 +39,9 @@ struct loc_network* loc_network_unref(struct loc_network* network); - char* loc_network_str(struct loc_network* network); - int loc_network_address_family(struct loc_network* network); +- int r = loc_database_enumerator_new(&enumerator, self->db, what); ++ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +@@ -223,7 +225,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en + } -+const struct in6_addr* loc_network_get_first_address(struct loc_network* network); - char* loc_network_format_first_address(struct loc_network* network); -+const struct in6_addr* loc_network_get_last_address(struct loc_network* network); - char* loc_network_format_last_address(struct loc_network* network); - int loc_network_match_address(struct loc_network* network, const struct in6_addr* address); + static PyObject* Database_ases(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0); + } -diff --git a/src/network.c b/src/network.c -index 28ca2df..4c8787a 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -343,10 +343,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc - return string; + static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { +@@ -234,7 +236,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { + + struct loc_database_enumerator* enumerator; + +- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES); ++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +@@ -250,44 +252,142 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { } -+LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) { -+ return &network->first_address; + static PyObject* Database_networks(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0); +} + - LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) { - return loc_network_format_address(network, &network->first_address); - } - -+LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) { -+ return &network->last_address; -+} ++static PyObject* Database_networks_flattened(DatabaseObject *self) { ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN); + } + + static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { +- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL }; +- const char* country_code = NULL; +- unsigned int asn = 0; ++ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL }; ++ PyObject* country_codes = NULL; ++ PyObject* asn_list = NULL; + int flags = 0; + int family = 0; ++ int flatten = 0; + +- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family)) ++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist, ++ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten)) + return NULL; + + struct loc_database_enumerator* enumerator; +- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS); ++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, ++ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; + } + + // Set country code we are searching for +- if (country_code) { +- r = loc_database_enumerator_set_country_code(enumerator, country_code); ++ if (country_codes) { ++ struct loc_country_list* countries; ++ r = loc_country_list_new(loc_ctx, &countries); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not create country list"); ++ return NULL; ++ } ++ ++ for (unsigned int i = 0; i < PyList_Size(country_codes); i++) { ++ PyObject* item = PyList_GetItem(country_codes, i); ++ ++ if (!PyUnicode_Check(item)) { ++ PyErr_SetString(PyExc_TypeError, "Country codes must be strings"); ++ loc_country_list_unref(countries); ++ return NULL; ++ } ++ ++ const char* country_code = PyUnicode_AsUTF8(item); ++ ++ struct loc_country* country; ++ r = loc_country_new(loc_ctx, &country, country_code); ++ if (r) { ++ if (r == -EINVAL) { ++ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code); ++ } else { ++ PyErr_SetString(PyExc_SystemError, "Could not create country"); ++ } ++ ++ loc_country_list_unref(countries); ++ return NULL; ++ } ++ ++ // Append it to the list ++ r = loc_country_list_append(countries, country); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not append country to the list"); + - LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) { - return loc_network_format_address(network, &network->last_address); - } -diff --git a/src/python/database.c b/src/python/database.c -index ed22275..f385c61 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -329,7 +329,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, ++ loc_country_list_unref(countries); ++ loc_country_unref(country); ++ return NULL; ++ } ++ ++ loc_country_unref(country); ++ } + ++ r = loc_database_enumerator_set_countries(enumerator, countries); if (r) { PyErr_SetFromErrno(PyExc_SystemError); - -- loc_as_list_unref(countries); ++ + loc_country_list_unref(countries); return NULL; } ++ ++ loc_country_list_unref(countries); + } -diff --git a/src/python/export.py b/src/python/export.py -index 5bc9f30..6b39878 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -142,11 +142,8 @@ class XTGeoIPOutputWriter(OutputWriter): - mode = "wb" - - def _write_network(self, network): -- for address in (network.first_address, network.last_address): -- # Convert this into a string of bits -- bytes = socket.inet_pton(network.family, address) -- -- self.f.write(bytes) -+ for address in (network._first_address, network._last_address): -+ self.f.write(address) - - - formats = { -diff --git a/src/python/network.c b/src/python/network.c -index ed91d65..742b472 100644 ---- a/src/python/network.c -+++ b/src/python/network.c -@@ -215,6 +215,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) { - return obj; - } + // Set the ASN we are searching for +- if (asn) { +- r = loc_database_enumerator_set_asn(enumerator, asn); ++ if (asn_list) { ++ struct loc_as_list* asns; ++ r = loc_as_list_new(loc_ctx, &asns); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not create AS list"); ++ return NULL; ++ } ++ ++ for (unsigned int i = 0; i < PyList_Size(asn_list); i++) { ++ PyObject* item = PyList_GetItem(asn_list, i); ++ ++ if (!PyLong_Check(item)) { ++ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers"); -+static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) { -+ struct in_addr address4; ++ loc_as_list_unref(asns); ++ return NULL; ++ } + -+ // Convert IPv4 addresses to struct in_addr -+ if (IN6_IS_ADDR_V4MAPPED(address6)) { -+ address4.s_addr = address6->s6_addr32[3]; ++ unsigned long number = PyLong_AsLong(item); + -+ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4)); -+ } ++ struct loc_as* as; ++ r = loc_as_new(loc_ctx, &as, number); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not create AS"); + -+ // Return IPv6 addresses as they are -+ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6)); -+} ++ loc_as_list_unref(asns); ++ loc_as_unref(as); ++ return NULL; ++ } + -+static PyObject* Network_get__first_address(NetworkObject* self) { -+ const struct in6_addr* address = loc_network_get_first_address(self->network); ++ r = loc_as_list_append(asns, as); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list"); + -+ return PyBytes_FromAddress(address); -+} ++ loc_as_list_unref(asns); ++ loc_as_unref(as); ++ return NULL; ++ } + - static PyObject* Network_get_last_address(NetworkObject* self) { - char* address = loc_network_format_last_address(self->network); ++ loc_as_unref(as); ++ } ++ ++ r = loc_database_enumerator_set_asns(enumerator, asns); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); ++ ++ loc_as_list_unref(asns); + return NULL; + } ++ ++ loc_as_list_unref(asns); + } -@@ -224,6 +244,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) { - return obj; + // Set the flags we are searching for +@@ -317,7 +417,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, } -+static PyObject* Network_get__last_address(NetworkObject* self) { -+ const struct in6_addr* address = loc_network_get_last_address(self->network); -+ -+ return PyBytes_FromAddress(address); -+} -+ - static struct PyMethodDef Network_methods[] = { - { - "exclude", -@@ -281,6 +307,13 @@ static struct PyGetSetDef Network_getsetters[] = { + static PyObject* Database_countries(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0); + } + + static struct PyMethodDef Database_methods[] = { +@@ -403,6 +503,13 @@ static struct PyGetSetDef Database_getsetters[] = { NULL, NULL, }, + { -+ "_first_address", -+ (getter)Network_get__first_address, ++ "networks_flattened", ++ (getter)Database_networks_flattened, + NULL, + NULL, + NULL, + }, { - "last_address", - (getter)Network_get_last_address, -@@ -288,6 +321,13 @@ static struct PyGetSetDef Network_getsetters[] = { - NULL, - NULL, - }, -+ { -+ "_last_address", -+ (getter)Network_get__last_address, -+ NULL, -+ NULL, -+ NULL, -+ }, - { NULL }, - }; + "vendor", + (getter)Database_get_vendor, +diff --git a/src/python/downloader.py b/src/python/downloader.py +index 87bbb68..05f7872 100644 +--- a/src/python/downloader.py ++++ b/src/python/downloader.py +@@ -119,8 +119,8 @@ class Downloader(object): + + headers = {} + if timestamp: +- headers["If-Modified-Since"] = timestamp.strftime( +- "%a, %d %b %Y %H:%M:%S GMT", ++ headers["If-Modified-Since"] = time.strftime( ++ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp), + ) + + t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) +@@ -195,7 +195,7 @@ class Downloader(object): + db = Database(f.name) + + # Database is not recent +- if timestamp and db.created_at < timestamp.timestamp(): ++ if timestamp and db.created_at < timestamp: + return False --- -2.20.1 - -From 90d2194a876c223f9124ce9e27bdee6a6b49ff6a Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 19 Nov 2020 12:40:01 +0000 -Subject: [PATCH 075/111] export: Remove old flattening feature - -The database enumerator now only returns networks that will -never overlap. - -Signed-off-by: Michael Tremer ---- - src/python/export.py | 34 ++++++---------------------------- - 1 file changed, 6 insertions(+), 28 deletions(-) - + log.info("Downloaded new database from %s" % (time.strftime( diff --git a/src/python/export.py b/src/python/export.py -index 6b39878..4702bcf 100644 +index d15c6f0..f0eae26 100644 --- a/src/python/export.py +++ b/src/python/export.py +@@ -29,7 +29,7 @@ import _location + log = logging.getLogger("location.export") + log.propagate = 1 + +-flags = { ++FLAGS = { + _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", + _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", + _location.NETWORK_FLAG_ANYCAST : "A3", @@ -39,11 +39,8 @@ class OutputWriter(object): suffix = "networks" mode = "w" @@ -7996,7 +2524,7 @@ index 6b39878..4702bcf 100644 def _write_header(self): """ The header of the file -@@ -84,15 +69,8 @@ class OutputWriter(object): +@@ -84,16 +69,8 @@ class OutputWriter(object): """ pass @@ -8005,15 +2533,16 @@ index 6b39878..4702bcf 100644 - def write(self, network): - if self.flatten and self._flatten(network): -- log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network)) +- log.debug("Skipping writing network %s" % network) - return - -- return self._write_network(network) +- # Write the network to file +- self._write_network(network) + self.f.write("%s\n" % network) def finish(self): """ -@@ -113,7 +91,7 @@ class IpsetOutputWriter(OutputWriter): +@@ -114,7 +91,7 @@ class IpsetOutputWriter(OutputWriter): def _write_header(self): self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix) @@ -8022,1756 +2551,875 @@ index 6b39878..4702bcf 100644 self.f.write("add %s %s\n" % (self.prefix, network)) -@@ -129,7 +107,7 @@ class NftablesOutputWriter(OutputWriter): +@@ -130,7 +107,7 @@ class NftablesOutputWriter(OutputWriter): def _write_footer(self): self.f.write("}\n") - def _write_network(self, network): + def write(self, network): - self.f.write(" %s,\n" % network) - - -@@ -141,7 +119,7 @@ class XTGeoIPOutputWriter(OutputWriter): - suffix = "iv" - mode = "wb" - -- def _write_network(self, network): -+ def write(self, network): - for address in (network._first_address, network._last_address): - self.f.write(address) - --- -2.20.1 - -From 90188dad86223380b0854b523b63ec024117c5f2 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 19 Nov 2020 12:41:19 +0000 -Subject: [PATCH 076/111] export: Speed-up export in xt_geoip format - -Removing the loop avoids creating a tuple and then iterating over it - -Signed-off-by: Michael Tremer ---- - src/python/export.py | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index 4702bcf..f0eae26 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -120,8 +120,8 @@ class XTGeoIPOutputWriter(OutputWriter): - mode = "wb" - - def write(self, network): -- for address in (network._first_address, network._last_address): -- self.f.write(address) -+ self.f.write(network._first_address) -+ self.f.write(network._last_address) - - - formats = { --- -2.20.1 - -From 6661692f3bc8f788af8b75ae25561f4cc4a2acb5 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Thu, 19 Nov 2020 12:48:46 +0000 -Subject: [PATCH 077/111] database: Disable some useless code when not running - in debug mode - -Signed-off-by: Michael Tremer ---- - src/database.c | 16 +++++++++++++++- - 1 file changed, 15 insertions(+), 1 deletion(-) - -diff --git a/src/database.c b/src/database.c -index 83dd752..ef4f505 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -619,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { - } - - clock_t end = clock(); -- DEBUG(db->ctx, "Signature checked in %.4fms\n", -+ INFO(db->ctx, "Signature checked in %.4fms\n", - (double)(end - start) / CLOCKS_PER_SEC * 1000); - - CLEANUP: -@@ -679,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, - off_t lo = 0; - off_t hi = db->as_count - 1; - -+#ifdef ENABLE_DEBUG - // Save start time - clock_t start = clock(); -+#endif - - while (lo <= hi) { - off_t i = (lo + hi) / 2; -@@ -693,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, - // Check if this is a match - uint32_t as_number = loc_as_get_number(*as); - if (as_number == number) { -+#ifdef ENABLE_DEBUG - clock_t end = clock(); - - // Log how fast this has been - DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number, - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif - - return 0; - } -@@ -741,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ - return -1; - } - -+#ifdef ENABLE_DEBUG - if (r == 0) { - char* string = loc_network_str(*network); - DEBUG(db->ctx, "Got network %s\n", string); - free(string); - } -+#endif - - return r; - } -@@ -840,17 +846,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db, - - *network = NULL; - -+#ifdef ENABLE_DEBUG - // Save start time - clock_t start = clock(); -+#endif + self.f.write(" %s,\n" % network) - int r = __loc_database_lookup(db, address, network, &network_address, - db->network_nodes_v1, 0); -+#ifdef ENABLE_DEBUG - clock_t end = clock(); +@@ -142,14 +119,9 @@ class XTGeoIPOutputWriter(OutputWriter): + suffix = "iv" + mode = "wb" - // Log how fast this has been - DEBUG(db->ctx, "Executed network search in %.4fms\n", - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif +- def _write_network(self, network): +- for address in (network.first_address, network.last_address): +- # Convert this into a string of bits +- bytes = socket.inet_pton( +- network.family, address, +- ) +- +- self.f.write(bytes) ++ def write(self, network): ++ self.f.write(network._first_address) ++ self.f.write(network._last_address) - return r; - } -@@ -897,8 +907,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, - off_t lo = 0; - off_t hi = db->countries_count - 1; -+#ifdef ENABLE_DEBUG - // Save start time - clock_t start = clock(); -+#endif + formats = { +@@ -185,8 +157,14 @@ class Exporter(object): - while (lo <= hi) { - off_t i = (lo + hi) / 2; -@@ -913,11 +925,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, - int result = strcmp(code, cc); + writers[asn] = self.writer.open(filename, prefix="AS%s" % asn) - if (result == 0) { -+#ifdef ENABLE_DEBUG - clock_t end = clock(); ++ # Filter countries from special country codes ++ country_codes = [ ++ country_code for country_code in countries if not country_code in FLAGS.values() ++ ] ++ + # Get all networks that match the family +- networks = self.db.search_networks(family=family) ++ networks = self.db.search_networks(family=family, ++ country_codes=country_codes, asns=asns, flatten=True) - // Log how fast this has been - DEBUG(db->ctx, "Found country %s in %.4fms\n", cc, - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif + # Walk through all networks + for network in networks: +@@ -203,10 +181,10 @@ class Exporter(object): + pass - return 0; - } --- -2.20.1 - -From a17c353bce55a9f38f67b4b7d2425194cfa208e7 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 20 Nov 2020 14:43:31 +0000 -Subject: [PATCH 078/111] test: Add tests to network-lists - -Signed-off-by: Michael Tremer ---- - Makefile.am | 10 ++++ - src/.gitignore | 1 + - src/test-network-list.c | 126 ++++++++++++++++++++++++++++++++++++++++ - 3 files changed, 137 insertions(+) - create mode 100644 src/test-network-list.c - -diff --git a/Makefile.am b/Makefile.am -index d0cc793..ebd7e17 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -318,6 +318,7 @@ check_PROGRAMS = \ - src/test-database \ - src/test-as \ - src/test-network \ -+ src/test-network-list \ - src/test-country \ - src/test-signature + # Handle flags +- for flag in flags: ++ for flag in FLAGS: + if network.has_flag(flag): + # Fetch the "fake" country code +- country = flags[flag] ++ country = FLAGS[flag] -@@ -357,6 +358,15 @@ src_test_network_CFLAGS = \ - src_test_network_LDADD = \ - src/libloc.la + try: + writers[country].write(network) +diff --git a/src/python/importer.py b/src/python/importer.py +index f19db4b..5f46bc3 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -64,7 +64,7 @@ EXTENDED_SOURCES = ( + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", -+src_test_network_list_SOURCES = \ -+ src/test-network-list.c -+ -+src_test_network_list_CFLAGS = \ -+ $(TESTS_CFLAGS) -+ -+src_test_network_list_LDADD = \ -+ src/libloc.la -+ - src_test_stringpool_SOURCES = \ - src/test-stringpool.c + # Latin America and Caribbean Network Information Centre +- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", ++ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", -diff --git a/src/.gitignore b/src/.gitignore -index caf80b5..3ccbdb8 100644 ---- a/src/.gitignore -+++ b/src/.gitignore -@@ -10,5 +10,6 @@ test-libloc - test-database - test-country - test-network -+test-network-list - test-signature - test-stringpool -diff --git a/src/test-network-list.c b/src/test-network-list.c -new file mode 100644 -index 0000000..3061d63 ---- /dev/null -+++ b/src/test-network-list.c -@@ -0,0 +1,126 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2017 IPFire Development Team -+ -+ This program is free software; you can redistribute it and/or modify -+ it under the terms of the GNU General Public License as published by -+ the Free Software Foundation; either version 2 of the License, or -+ (at your option) any later version. -+ -+ This program is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ GNU General Public License for more details. -+*/ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+ -+int main(int argc, char** argv) { -+ int err; -+ -+ struct loc_ctx* ctx; -+ err = loc_new(&ctx); -+ if (err < 0) -+ exit(EXIT_FAILURE); -+ -+ // Enable debug logging -+ loc_set_log_priority(ctx, LOG_DEBUG); -+ -+ // Create a network -+ struct loc_network* network1; -+ err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32"); -+ if (err) { -+ fprintf(stderr, "Could not create the network1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet1; -+ err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet2; -+ err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet2\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Make a list with both subnets -+ struct loc_network_list* subnets; -+ err = loc_network_list_new(ctx, &subnets); -+ if (err) { -+ fprintf(stderr, "Could not create subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ size_t size = loc_network_list_size(subnets); -+ if (size > 0) { -+ fprintf(stderr, "The list is not empty: %zu\n", size); -+ exit(EXIT_FAILURE); -+ } -+ -+ err = loc_network_list_push(subnets, subnet1); -+ if (err) { -+ fprintf(stderr, "Could not add subnet1 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (loc_network_list_empty(subnets)) { -+ fprintf(stderr, "The subnets list reports that it is empty\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ err = loc_network_list_push(subnets, subnet2); -+ if (err) { -+ fprintf(stderr, "Could not add subnet2 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ size = loc_network_list_size(subnets); -+ if (size != 2) { -+ fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Exclude subnet1 from network1 -+ struct loc_network_list* excluded = loc_network_exclude(network1, subnet1); -+ if (!excluded) { -+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ -+ // Exclude all subnets from network1 -+ excluded = loc_network_exclude_list(network1, subnets); -+ if (!excluded) { -+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ -+ if (excluded) -+ loc_network_list_unref(excluded); -+ -+ loc_network_list_unref(subnets); -+ loc_network_unref(network1); -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); -+ loc_unref(ctx); -+ -+ return EXIT_SUCCESS; -+} --- -2.20.1 - -From 1209ff0cd4be6b2f52a5e82168db8a8ac68e628a Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 20 Nov 2020 16:24:40 +0000 -Subject: [PATCH 079/111] network: Fix loc_network_is_subnet() - -This function always returned false. - -Signed-off-by: Michael Tremer ---- - src/network.c | 16 +++------------- - 1 file changed, 3 insertions(+), 13 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 4c8787a..b440d76 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -491,12 +491,12 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network - LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. -- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) -+ if (in6_addr_cmp(&self->first_address, &other->first_address) > 0) - return 0; + # Réseaux IP Européens + #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 1467923..2dec89e 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -152,6 +152,7 @@ class CLI(object): + last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); + CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network); + CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network)); ++ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops); + + -- autnums + CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); +@@ -165,6 +166,7 @@ class CLI(object): + -- networks + CREATE TABLE IF NOT EXISTS networks(network inet, country text); + CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); ++ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); + CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); - // If the end address of the other network is greater than this network, - // it cannot be a subnet. -- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) -+ if (in6_addr_cmp(&self->last_address, &other->last_address) < 0) - return 0; + -- overrides +@@ -188,6 +190,8 @@ class CLI(object): + ); + CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network + ON network_overrides(network); ++ CREATE INDEX IF NOT EXISTS network_overrides_search ++ ON network_overrides USING GIST(network inet_ops); + """) - return 1; -@@ -504,17 +504,7 @@ LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_networ + return db +@@ -234,32 +238,24 @@ class CLI(object): - // XXX DEPRECATED - I find this too difficult to use - LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { -- // If the start address of the other network is smaller than this network, -- // it cannot be a subnet. -- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) -- return 0; -- -- // If the end address of the other network is greater than this network, -- // it cannot be a subnet. -- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) -- return 0; + # Select all known networks + rows = self.db.query(""" +- -- Get a (sorted) list of all known networks +- WITH known_networks AS ( +- SELECT network FROM announcements +- UNION +- SELECT network FROM networks +- ORDER BY network +- ) - -- return 1; -+ return loc_network_is_subnet(other, self); - } - - LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) { --- -2.20.1 - -From 92349c14876f8e4711739c82d1a389a53bcb27a5 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 20 Nov 2020 16:25:27 +0000 -Subject: [PATCH 080/111] network: Remove debugging output - -Signed-off-by: Michael Tremer ---- - src/network.c | 4 ---- - 1 file changed, 4 deletions(-) - -diff --git a/src/network.c b/src/network.c -index b440d76..3271272 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -670,10 +670,6 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude( - loc_network_unref(subnet2); - } + -- Return a list of those networks enriched with all + -- other information that we store in the database + SELECT +- DISTINCT ON (known_networks.network) +- known_networks.network AS network, +- announcements.autnum AS autnum, ++ DISTINCT ON (network) ++ network, ++ autnum, --#ifdef ENABLE_DEBUG -- loc_network_list_dump(list); --#endif + -- Country + COALESCE( + ( + SELECT country FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT country FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + networks.country + ) AS country, +@@ -268,50 +264,67 @@ class CLI(object): + COALESCE( + ( + SELECT is_anonymous_proxy FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anonymous_proxy FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_anonymous_proxy, + COALESCE( + ( + SELECT is_satellite_provider FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_satellite_provider FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_satellite_provider, + COALESCE( + ( + SELECT is_anycast FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anycast FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE +- ) AS is_anycast, - - // Return the result - return list; - --- -2.20.1 - -From ed0f53df0f3ebb915faf25138cc09df7555415a3 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 20 Nov 2020 16:25:56 +0000 -Subject: [PATCH 081/111] database: Flatten out code due to compiler errors - -It looks like GCC could not follow the code as it was -written. Therefore I clean up more often now instead -of having a GOTO block where I do that in. - -Signed-off-by: Michael Tremer ---- - src/database.c | 59 ++++++++++++++++++++++++++++++++++---------------- - 1 file changed, 40 insertions(+), 19 deletions(-) - -diff --git a/src/database.c b/src/database.c -index ef4f505..c21b957 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1305,8 +1305,12 @@ static int __loc_database_enumerator_next_network_flattened( - while (1) { - // Fetch the next network in line - r = __loc_database_enumerator_next_network(enumerator, &subnet, 0); -- if (r) -- goto END; -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); +- -- Must be part of returned values for ORDER BY clause +- masklen(announcements.network) AS sort_a, +- masklen(networks.network) AS sort_b +- FROM known_networks +- LEFT JOIN announcements ON known_networks.network <<= announcements.network +- LEFT JOIN networks ON known_networks.network <<= networks.network +- ORDER BY known_networks.network, sort_a DESC, sort_b DESC ++ ) AS is_anycast ++ FROM ( ++ SELECT ++ known_networks.network AS network, ++ announcements.autnum AS autnum, ++ networks.country AS country, + -+ return r; -+ } ++ -- Must be part of returned values for ORDER BY clause ++ masklen(announcements.network) AS sort_a, ++ masklen(networks.network) AS sort_b ++ FROM ( ++ SELECT network FROM announcements ++ UNION ALL ++ SELECT network FROM networks ++ UNION ALL ++ SELECT network FROM network_overrides ++ ) known_networks ++ LEFT JOIN ++ announcements ON known_networks.network <<= announcements.network ++ LEFT JOIN ++ networks ON known_networks.network <<= networks.network ++ ORDER BY ++ known_networks.network, ++ sort_a DESC, ++ sort_b DESC ++ ) networks + """) - // End if we did not receive another subnet - if (!subnet) -@@ -1315,8 +1319,12 @@ static int __loc_database_enumerator_next_network_flattened( - // Collect all subnets in a list - if (loc_network_is_subnet(*network, subnet)) { - r = loc_network_list_push(subnets, subnet); -- if (r) -- goto END; -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); + for row in rows: +@@ -363,6 +376,16 @@ class CLI(object): + CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) + ON COMMIT DROP; + CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); + -+ return r; -+ } ++ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) ++ ON COMMIT DROP; ++ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); ++ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); ++ """) ++ ++ # Remove all previously imported content ++ self.db.execute(""" ++ TRUNCATE TABLE networks; + """) - loc_network_unref(subnet); - continue; -@@ -1324,8 +1332,12 @@ static int __loc_database_enumerator_next_network_flattened( + for source in location.importer.WHOIS_SOURCES: +@@ -370,31 +393,72 @@ class CLI(object): + for block in f: + self._parse_block(block) - // If this is not a subnet, we push it back onto the stack and break - r = loc_network_list_push(enumerator->stack, subnet); -- if (r) -- goto END; -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); ++ # Process all parsed networks from every RIR we happen to have access to, ++ # insert the largest network chunks into the networks table immediately... ++ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") + -+ return r; -+ } - - loc_network_unref(subnet); - break; -@@ -1343,29 +1355,38 @@ static int __loc_database_enumerator_next_network_flattened( - // If the network has any subnets, we will break it into smaller parts - // without the subnets. - struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets); -- if (!excluded || loc_network_list_empty(excluded)) { -- r = 1; -- goto END; -+ if (!excluded) { -+ loc_network_list_unref(subnets); -+ return -1; -+ } ++ for family in (row.family for row in families): ++ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) + -+ // Merge excluded list with subnets -+ r = loc_network_list_merge(subnets, excluded); -+ if (r) { -+ loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); ++ self.db.execute("INSERT INTO networks(network, country) \ ++ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) + -+ return r; - } - -+ // We no longer need the excluded list -+ loc_network_list_unref(excluded); ++ # ... determine any other prefixes for this network family, ... ++ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ ++ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) + -+ // Sort all subnets -+ loc_network_list_sort(subnets); ++ # ... and insert networks with this prefix in case they provide additional ++ # information (i. e. subnet of a larger chunk with a different country) ++ for prefix in (row.prefix for row in prefixes): ++ self.db.execute(""" ++ WITH candidates AS ( ++ SELECT ++ _rirdata.network, ++ _rirdata.country ++ FROM ++ _rirdata ++ WHERE ++ family(_rirdata.network) = %s ++ AND ++ masklen(_rirdata.network) = %s ++ ), ++ filtered AS ( ++ SELECT ++ DISTINCT ON (c.network) ++ c.network, ++ c.country, ++ masklen(networks.network), ++ networks.country AS parent_country ++ FROM ++ candidates c ++ LEFT JOIN ++ networks ++ ON ++ c.network << networks.network ++ ORDER BY ++ c.network, ++ masklen(networks.network) DESC NULLS LAST ++ ) ++ INSERT INTO ++ networks(network, country) ++ SELECT ++ network, ++ country ++ FROM ++ filtered ++ WHERE ++ parent_country IS NULL ++ OR ++ country <> parent_country ++ ON CONFLICT DO NOTHING""", ++ family, prefix, ++ ) + - // Replace network with the first one - loc_network_unref(*network); - -- *network = loc_network_list_pop_first(excluded); -+ *network = loc_network_list_pop_first(subnets); - - // Push the rest onto the stack -- loc_network_list_reverse(excluded); -- loc_network_list_merge(enumerator->stack, excluded); + self.db.execute(""" + INSERT INTO autnums(number, name) + SELECT _autnums.number, _organizations.name FROM _autnums + JOIN _organizations ON _autnums.organization = _organizations.handle +- ON CONFLICT (number) DO UPDATE SET name = excluded.name +- """) +- +- self.db.execute(""" +- --- Purge any redundant entries +- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP +- AS +- SELECT network FROM networks candidates +- WHERE EXISTS ( +- SELECT FROM networks +- WHERE +- networks.network << candidates.network +- AND +- networks.country = candidates.country +- ); - -- loc_network_list_unref(excluded); +- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network); - --END: -- if (subnet) -- loc_network_unref(subnet); -+ loc_network_list_reverse(subnets); -+ loc_network_list_merge(enumerator->stack, subnets); - - loc_network_list_unref(subnets); - -- return r; -+ return 0; - } - - LOC_EXPORT int loc_database_enumerator_next_network( --- -2.20.1 - -From a1024390795d60fab9f697fa230c9901ebd7d221 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 20 Nov 2020 16:42:55 +0000 -Subject: [PATCH 082/111] network-list: Implement merging in reverse in one - step - -This will save us some time because we do not need to -change the list in place first and then merge it. - -Signed-off-by: Michael Tremer ---- - src/database.c | 3 +-- - src/libloc.sym | 1 + - src/loc/network-list.h | 1 + - src/network-list.c | 13 +++++++++++++ - 4 files changed, 16 insertions(+), 2 deletions(-) - -diff --git a/src/database.c b/src/database.c -index c21b957..ba4f98a 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1381,8 +1381,7 @@ static int __loc_database_enumerator_next_network_flattened( - *network = loc_network_list_pop_first(subnets); - - // Push the rest onto the stack -- loc_network_list_reverse(subnets); -- loc_network_list_merge(enumerator->stack, subnets); -+ loc_network_list_merge_reverse(enumerator->stack, subnets); - - loc_network_list_unref(subnets); - -diff --git a/src/libloc.sym b/src/libloc.sym -index 406dd15..56cada8 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -140,6 +140,7 @@ global: - loc_network_list_empty; - loc_network_list_get; - loc_network_list_merge; -+ loc_network_list_merge_reverse; - loc_network_list_new; - loc_network_list_pop; - loc_network_list_pop_first; -diff --git a/src/loc/network-list.h b/src/loc/network-list.h -index af3b28d..89776a6 100644 ---- a/src/loc/network-list.h -+++ b/src/loc/network-list.h -@@ -33,5 +33,6 @@ int loc_network_list_contains(struct loc_network_list* list, struct loc_network* - void loc_network_list_sort(struct loc_network_list* list); - void loc_network_list_reverse(struct loc_network_list* list); - int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); -+int loc_network_list_merge_reverse(struct loc_network_list* self, struct loc_network_list* other); +- DELETE FROM networks WHERE EXISTS ( +- SELECT FROM _garbage WHERE networks.network = _garbage.network +- ); ++ ON CONFLICT (number) DO UPDATE SET name = excluded.name; + """) - #endif -diff --git a/src/network-list.c b/src/network-list.c -index 9cb4547..b455caf 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -242,3 +242,16 @@ LOC_EXPORT int loc_network_list_merge( + # Download all extended sources +@@ -405,6 +469,69 @@ class CLI(object): + for line in f: + self._parse_line(line) - return 0; - } ++ def _check_parsed_network(self, network): ++ """ ++ Assistive function to detect and subsequently sort out parsed ++ networks from RIR data (both Whois and so-called "extended sources"), ++ which are or have... + -+LOC_EXPORT int loc_network_list_merge_reverse( -+ struct loc_network_list* self, struct loc_network_list* other) { -+ int r; ++ (a) not globally routable (RFC 1918 space, et al.) ++ (b) covering a too large chunk of the IP address space (prefix length ++ is < 7 for IPv4 networks, and < 10 for IPv6) ++ (c) "0.0.0.0" or "::" as a network address ++ (d) are too small for being publicly announced (we have decided not to ++ process them at the moment, as they significantly enlarge our ++ database without providing very helpful additional information) + -+ for (int i = other->size - 1; i >= 0; i--) { -+ r = loc_network_list_push(self, other->elements[i]); -+ if (r) -+ return r; -+ } ++ This unfortunately is necessary due to brain-dead clutter across ++ various RIR databases, causing mismatches and eventually disruptions. + -+ return 0; -+} --- -2.20.1 - -From a5967330d530504db401540d4bcd5474fe00e421 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 20 Nov 2020 18:36:48 +0000 -Subject: [PATCH 083/111] network: Optimise _subnet function - -This function used to create a network list which always -had exactly two elements. Since splitting a network in half -always returns two parts, we can simply return them as a -pointer. - -This improves returning the network tree by about 17%. - -Signed-off-by: Michael Tremer ---- - src/loc/network.h | 2 +- - src/network.c | 158 ++++++++++++++++++--------------------------- - src/test-network.c | 51 ++++++++------- - 3 files changed, 91 insertions(+), 120 deletions(-) - -diff --git a/src/loc/network.h b/src/loc/network.h -index 4b7410c..d67ec54 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -62,7 +62,7 @@ int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); --struct loc_network_list* loc_network_subnets(struct loc_network* network); -+int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2); - struct loc_network_list* loc_network_exclude( - struct loc_network* self, struct loc_network* other); - struct loc_network_list* loc_network_exclude_list( -diff --git a/src/network.c b/src/network.c -index 3271272..ff9c1eb 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -507,66 +507,91 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net - return loc_network_is_subnet(other, self); - } - --LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) { -- struct loc_network_list* list; -+LOC_EXPORT int loc_network_subnets(struct loc_network* network, -+ struct loc_network** subnet1, struct loc_network** subnet2) { -+ int r; -+ *subnet1 = NULL; -+ *subnet2 = NULL; - - // New prefix length - unsigned int prefix = network->prefix + 1; - - // Check if the new prefix is valid - if (valid_prefix(&network->first_address, prefix)) -- return NULL; -- -- // Create a new list with the result -- int r = loc_network_list_new(network->ctx, &list); -- if (r) { -- ERROR(network->ctx, "Could not create network list: %d\n", r); -- return NULL; -- } -- -- struct loc_network* subnet1 = NULL; -- struct loc_network* subnet2 = NULL; -+ return -1; - - // Create the first half of the network -- r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix); -+ r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix); - if (r) -- goto ERROR; -+ return r; - - // The next subnet starts after the first one -- struct in6_addr first_address = address_increment(&subnet1->last_address); -+ struct in6_addr first_address = address_increment(&(*subnet1)->last_address); - - // Create the second half of the network -- r = loc_network_new(network->ctx, &subnet2, &first_address, prefix); -- if (r) -- goto ERROR; -- -- // Push the both onto the stack (in reverse order) -- r = loc_network_list_push(list, subnet2); -+ r = loc_network_new(network->ctx, subnet2, &first_address, prefix); - if (r) -- goto ERROR; -- -- r = loc_network_list_push(list, subnet1); -- if (r) -- goto ERROR; -+ return r; - - // Copy country code - const char* country_code = loc_network_get_country_code(network); - if (country_code) { -- loc_network_set_country_code(subnet1, country_code); -- loc_network_set_country_code(subnet2, country_code); -+ loc_network_set_country_code(*subnet1, country_code); -+ loc_network_set_country_code(*subnet2, country_code); - } - - // Copy ASN - uint32_t asn = loc_network_get_asn(network); - if (asn) { -- loc_network_set_asn(subnet1, asn); -- loc_network_set_asn(subnet2, asn); -+ loc_network_set_asn(*subnet1, asn); -+ loc_network_set_asn(*subnet2, asn); - } - -- loc_network_unref(subnet1); -- loc_network_unref(subnet2); -+ return 0; -+} - -- return list; -+static int __loc_network_exclude(struct loc_network* network, -+ struct loc_network* other, struct loc_network_list* list) { -+ struct loc_network* subnet1 = NULL; -+ struct loc_network* subnet2 = NULL; ++ We will return False in case a network is not suitable for adding ++ it to our database, and True otherwise. ++ """ + -+ int r = loc_network_subnets(network, &subnet1, &subnet2); -+ if (r) -+ goto ERROR; ++ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)): ++ return False + -+ if (loc_network_eq(other, subnet1)) { -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; ++ if not network.is_global: ++ log.warning("Skipping non-globally routable network: %s" % network) ++ return False + -+ } else if (loc_network_eq(other, subnet2)) { -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; ++ if network.version == 4: ++ if network.prefixlen < 7: ++ log.warning("Skipping too big IP chunk: %s" % network) ++ return False + -+ } else if (loc_network_is_subnet_of(other, subnet1)) { -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; ++ if network.prefixlen > 24: ++ log.debug("Skipping network too small to be publicly announced: %s" % network) ++ return False + -+ r = __loc_network_exclude(subnet1, other, list); -+ if (r) -+ goto ERROR; ++ if str(network.network_address) == "0.0.0.0": ++ log.warning("Skipping network based on 0.0.0.0: %s" % network) ++ return False + -+ } else if (loc_network_is_subnet_of(other, subnet2)) { -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; ++ elif network.version == 6: ++ if network.prefixlen < 10: ++ log.warning("Skipping too big IP chunk: %s" % network) ++ return False + -+ r = __loc_network_exclude(subnet2, other, list); -+ if (r) -+ goto ERROR; ++ if network.prefixlen > 48: ++ log.debug("Skipping network too small to be publicly announced: %s" % network) ++ return False + -+ } else { -+ ERROR(network->ctx, "We should never get here\n"); -+ r = 1; -+ goto ERROR; -+ } - - ERROR: - if (subnet1) -@@ -575,10 +600,7 @@ ERROR: - if (subnet2) - loc_network_unref(subnet2); - -- if (list) -- loc_network_list_unref(list); -- -- return NULL; -+ return r; - } - - LOC_EXPORT struct loc_network_list* loc_network_exclude( -@@ -623,67 +645,15 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude( - return NULL; - } - -- struct loc_network_list* subnets = loc_network_subnets(self); -- -- struct loc_network* subnet1 = NULL; -- struct loc_network* subnet2 = NULL; -- -- while (subnets) { -- // Fetch both subnets -- subnet1 = loc_network_list_get(subnets, 0); -- subnet2 = loc_network_list_get(subnets, 1); -- -- // Free list -- loc_network_list_unref(subnets); -- subnets = NULL; -- -- if (loc_network_eq(other, subnet1)) { -- r = loc_network_list_push(list, subnet2); -- if (r) -- goto ERROR; -- -- } else if (loc_network_eq(other, subnet2)) { -- r = loc_network_list_push(list, subnet1); -- if (r) -- goto ERROR; -- -- } else if (loc_network_is_subnet_of(other, subnet1)) { -- r = loc_network_list_push(list, subnet2); -- if (r) -- goto ERROR; -- -- subnets = loc_network_subnets(subnet1); -- -- } else if (loc_network_is_subnet_of(other, subnet2)) { -- r = loc_network_list_push(list, subnet1); -- if (r) -- goto ERROR; -- -- subnets = loc_network_subnets(subnet2); -- -- } else { -- ERROR(self->ctx, "We should never get here\n"); -- goto ERROR; -- } -+ r = __loc_network_exclude(self, other, list); -+ if (r) { -+ loc_network_list_unref(list); ++ if str(network.network_address) == "::": ++ log.warning("Skipping network based on '::': %s" % network) ++ return False ++ ++ else: ++ # This should not happen... ++ log.warning("Skipping network of unknown family, this should not happen: %s" % network) ++ return False ++ ++ # In case we have made it here, the network is considered to ++ # be suitable for libloc consumption... ++ return True ++ + def _parse_block(self, block): + # Get first line to find out what type of block this is + line = block[0] +@@ -433,7 +560,7 @@ class CLI(object): + autnum["asn"] = m.group(2) -- loc_network_unref(subnet1); -- loc_network_unref(subnet2); -+ return NULL; - } + elif key == "org": +- autnum[key] = val ++ autnum[key] = val.upper() - // Return the result - return list; -- --ERROR: -- if (subnet1) -- loc_network_unref(subnet1); -- -- if (subnet2) -- loc_network_unref(subnet2); -- -- if (list) -- loc_network_list_unref(list); -- -- return NULL; - } + # Skip empty objects + if not autnum: +@@ -447,15 +574,22 @@ class CLI(object): + ) - LOC_EXPORT struct loc_network_list* loc_network_exclude_list( -diff --git a/src/test-network.c b/src/test-network.c -index 7c90224..79c2967 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -138,47 +138,48 @@ int main(int argc, char** argv) { - } + def _parse_inetnum_block(self, block): +- logging.debug("Parsing inetnum block:") ++ log.debug("Parsing inetnum block:") - // Check subnet function -- err = loc_network_is_subnet_of(network1, network2); -- if (err != 0) { -+ err = loc_network_is_subnet(network1, network2); -+ if (!err) { - fprintf(stderr, "Subnet check 1 failed: %d\n", err); - exit(EXIT_FAILURE); - } + inetnum = {} + for line in block: +- logging.debug(line) ++ log.debug(line) -- err = loc_network_is_subnet_of(network2, network1); -- if (err != 1) { -+ err = loc_network_is_subnet(network2, network1); -+ if (err) { - fprintf(stderr, "Subnet check 2 failed: %d\n", err); - exit(EXIT_FAILURE); - } + # Split line + key, val = split_line(line) -- // Make list of subnets -- struct loc_network_list* subnets = loc_network_subnets(network1); -- if (!subnets) { -- fprintf(stderr, "Could not find subnets of network\n"); -+ // Make subnets -+ struct loc_network* subnet1 = NULL; -+ struct loc_network* subnet2 = NULL; ++ # Filter any inetnum records which are only referring to IP space ++ # not managed by that specific RIR... ++ if key == "netname": ++ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()): ++ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip()) ++ return + -+ err = loc_network_subnets(network1, &subnet1, &subnet2); -+ if (err || !subnet1 || !subnet2) { -+ fprintf(stderr, "Could not find subnets of network: %d\n", err); - exit(EXIT_FAILURE); - } + if key == "inetnum": + start_address, delim, end_address = val.partition("-") + +@@ -467,7 +601,7 @@ class CLI(object): + start_address = ipaddress.ip_address(start_address) + end_address = ipaddress.ip_address(end_address) + except ValueError: +- logging.warning("Could not parse line: %s" % line) ++ log.warning("Could not parse line: %s" % line) + return + + # Set prefix to default +@@ -484,23 +618,24 @@ class CLI(object): + inetnum[key] = val -- loc_network_list_dump(subnets); + elif key == "country": +- if val == "UNITED STATES": +- val = "US" - -- while (!loc_network_list_empty(subnets)) { -- struct loc_network* subnet = loc_network_list_pop(subnets); -- if (!subnet) { -- fprintf(stderr, "Received an empty subnet\n"); -- exit(EXIT_FAILURE); -- } -+ char* s = loc_network_str(subnet1); -+ printf("Received subnet1 = %s\n", s); -+ free(s); + inetnum[key] = val.upper() -- char* s = loc_network_str(subnet); -- printf("Received subnet %s\n", s); -- free(s); -+ s = loc_network_str(subnet2); -+ printf("Received subnet2 = %s\n", s); -+ free(s); + # Skip empty objects +- if not inetnum: ++ if not inetnum or not "country" in inetnum: ++ return ++ ++ # Skip objects with bogus country code 'ZZ' ++ if inetnum.get("country") == "ZZ": ++ log.warning("Skipping network with bogus country 'ZZ': %s" % \ ++ (inetnum.get("inet6num") or inetnum.get("inetnum"))) + return -- if (!loc_network_is_subnet_of(subnet, network1)) { -- fprintf(stderr, "Not a subnet\n"); -- exit(EXIT_FAILURE); -- } -+ if (!loc_network_is_subnet(network1, subnet1)) { -+ fprintf(stderr, "Subnet1 is not a subnet\n"); -+ exit(EXIT_FAILURE); -+ } + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) -- loc_network_unref(subnet); -+ if (!loc_network_is_subnet(network1, subnet2)) { -+ fprintf(stderr, "Subnet2 is not a subnet\n"); -+ exit(EXIT_FAILURE); - } +- # Bail out in case we have processed a non-public IP network +- if network.is_private: +- logging.warning("Skipping non-globally routable network: %s" % network) ++ if not self._check_parsed_network(network): + return -- loc_network_list_unref(subnets); -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); +- self.db.execute("INSERT INTO networks(network, country) \ ++ self.db.execute("INSERT INTO _rirdata(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", + "%s" % network, inetnum.get("country"), + ) +@@ -511,7 +646,9 @@ class CLI(object): + # Split line + key, val = split_line(line) - struct loc_network_list* excluded = loc_network_exclude(network1, network2); - if (!excluded) { --- -2.20.1 - -From 7fe6a21845edf6692d239f228117bc95620d0419 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 20 Nov 2020 18:38:47 +0000 -Subject: [PATCH 084/111] network: Add function to return the prefix - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 12 ++++++++++++ - 3 files changed, 14 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 56cada8..9f41c89 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -125,6 +125,7 @@ global: - loc_network_new; - loc_network_new_from_string; - loc_network_overlaps; -+ loc_network_prefix; - loc_network_ref; - loc_network_set_asn; - loc_network_set_country_code; -diff --git a/src/loc/network.h b/src/loc/network.h -index d67ec54..7b2ae4c 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -38,6 +38,7 @@ struct loc_network* loc_network_ref(struct loc_network* network); - struct loc_network* loc_network_unref(struct loc_network* network); - char* loc_network_str(struct loc_network* network); - int loc_network_address_family(struct loc_network* network); -+unsigned int loc_network_prefix(struct loc_network* network); +- if key in ("organisation", "org-name"): ++ if key == "organisation": ++ org[key] = val.upper() ++ elif key == "org-name": + org[key] = val - const struct in6_addr* loc_network_get_first_address(struct loc_network* network); - char* loc_network_format_first_address(struct loc_network* network); -diff --git a/src/network.c b/src/network.c -index ff9c1eb..4720503 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -310,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) { - return network->family; - } + # Skip empty objects +@@ -581,6 +718,9 @@ class CLI(object): + log.warning("Invalid IP address: %s" % address) + return -+LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) { -+ switch (network->family) { -+ case AF_INET6: -+ return network->prefix; -+ -+ case AF_INET: -+ return network->prefix - 96; -+ } -+ -+ return 0; -+} ++ if not self._check_parsed_network(network): ++ return + - static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) { - const size_t length = INET6_ADDRSTRLEN; - --- -2.20.1 - -From cf8d3c6454843943e3bc81eb85522779d1c11f9b Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 20 Nov 2020 18:39:13 +0000 -Subject: [PATCH 085/111] network: Speed up subnet check - -There is no point in checking different address families -with each other and we do not need to compare addresses -when the prefix of the subnet does not fit into the -network to check. - -Signed-off-by: Michael Tremer ---- - src/network.c | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/src/network.c b/src/network.c -index 4720503..e52b58c 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -501,6 +501,14 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network - } + self.db.execute("INSERT INTO networks(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO \ + UPDATE SET country = excluded.country", +diff --git a/src/python/location.in b/src/python/location.in +index 44ad726..b30beae 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -253,6 +253,7 @@ class CLI(object): + network = db.lookup(address) + except ValueError: + print(_("Invalid IP address: %s") % address, file=sys.stderr) ++ return 2 - LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { -+ // Check family -+ if (self->family != other->family) -+ return 0; -+ -+ // The prefix must be smaller (this avoids the more complex comparisons later) -+ if (self->prefix > other->prefix) -+ return 0; -+ - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. - if (in6_addr_cmp(&self->first_address, &other->first_address) > 0) --- -2.20.1 - -From d6a5092f969bb3bd50d130d4ba64b4e4be2e61f6 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Fri, 20 Nov 2020 18:44:42 +0000 -Subject: [PATCH 086/111] network: Remove deprecated loc_network_is_subnet_of - function - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 1 - - src/loc/network.h | 1 - - src/network.c | 13 ++++--------- - src/python/network.c | 2 +- - 4 files changed, 5 insertions(+), 12 deletions(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 9f41c89..b2d8a31 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -118,7 +118,6 @@ global: - loc_network_gt; - loc_network_has_flag; - loc_network_is_subnet; -- loc_network_is_subnet_of; - loc_network_match_asn; - loc_network_match_country_code; - loc_network_match_flag; -diff --git a/src/loc/network.h b/src/loc/network.h -index 7b2ae4c..b31c8a2 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -62,7 +62,6 @@ int loc_network_eq(struct loc_network* self, struct loc_network* other); - int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); --int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); - int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2); - struct loc_network_list* loc_network_exclude( - struct loc_network* self, struct loc_network* other); -diff --git a/src/network.c b/src/network.c -index e52b58c..72b77e6 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -522,11 +522,6 @@ LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_networ - return 1; - } + args = { + "address" : address, +@@ -398,10 +399,7 @@ class CLI(object): --// XXX DEPRECATED - I find this too difficult to use --LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { -- return loc_network_is_subnet(other, self); --} + def handle_update(self, db, ns): + if ns.cron and db: +- now = datetime.datetime.utcnow() - - LOC_EXPORT int loc_network_subnets(struct loc_network* network, - struct loc_network** subnet1, struct loc_network** subnet2) { - int r; -@@ -589,7 +584,7 @@ static int __loc_network_exclude(struct loc_network* network, - if (r) - goto ERROR; - -- } else if (loc_network_is_subnet_of(other, subnet1)) { -+ } else if (loc_network_is_subnet(subnet1, other)) { - r = loc_network_list_push(list, subnet2); - if (r) - goto ERROR; -@@ -598,7 +593,7 @@ static int __loc_network_exclude(struct loc_network* network, - if (r) - goto ERROR; - -- } else if (loc_network_is_subnet_of(other, subnet2)) { -+ } else if (loc_network_is_subnet(subnet2, other)) { - r = loc_network_list_push(list, subnet1); - if (r) - goto ERROR; -@@ -645,7 +640,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude( - } - - // Other must be a subnet of self -- if (!loc_network_is_subnet_of(other, self)) { -+ if (!loc_network_is_subnet(self, other)) { - DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); +- # Parse the database timestamp +- t = datetime.datetime.utcfromtimestamp(db.created_at) ++ now = time.time() - return NULL; -@@ -726,7 +721,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - } + if ns.cron == "daily": + delta = datetime.timedelta(days=1) +@@ -410,22 +408,20 @@ class CLI(object): + elif ns.cron == "monthly": + delta = datetime.timedelta(days=30) - // Drop this subnet if is a subnet of another subnet -- if (loc_network_is_subnet_of(subnet, subnet_to_check)) { -+ if (loc_network_is_subnet(subnet_to_check, subnet)) { - passed = 0; - loc_network_unref(subnet); - break; -diff --git a/src/python/network.c b/src/python/network.c -index 742b472..b6e92fb 100644 ---- a/src/python/network.c -+++ b/src/python/network.c -@@ -194,7 +194,7 @@ static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { - if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) - return NULL; ++ delta = delta.total_seconds() ++ + # Check if the database has recently been updated +- if t >= (now - delta): ++ if db.created_at >= (now - delta): + log.info( +- _("The database has been updated recently (%s)") % \ +- format_timedelta(now - t), ++ _("The database has been updated recently"), + ) + return 3 -- if (loc_network_is_subnet_of(self->network, other->network)) -+ if (loc_network_is_subnet(other->network, self->network)) - Py_RETURN_TRUE; + # Fetch the timestamp we need from DNS + t = location.discover_latest_version() - Py_RETURN_FALSE; --- -2.20.1 - -From af4689bf5c56ad79e9e90396b41be460e49ef384 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 24 Nov 2020 14:55:08 +0000 -Subject: [PATCH 087/111] network-list: Make this a sorted list - -The list is now organised so that it is always ordered. - -This allows us to find if a network is on the list a lot -quicker using binary search as well as inserting a new -network at the right place. - -This will benefit loc_network_exclude with very large networks. - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network-list.c | 84 +++++++++++++++++++++++++++++++++++++++++----- - src/network.c | 22 ++++++++++++ - 4 files changed, 99 insertions(+), 9 deletions(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index b2d8a31..28cc8e8 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -106,6 +106,7 @@ global: +- # Parse timestamp into datetime format +- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None +- + # Check the version of the local database +- if db and timestamp and db.created_at >= timestamp.timestamp(): ++ if db and t and db.created_at >= t: + log.info("Already on the latest version") + return - # Network - loc_network_address_family; -+ loc_network_cmp; - loc_network_eq; - loc_network_exclude; - loc_network_exclude_list; -diff --git a/src/loc/network.h b/src/loc/network.h -index b31c8a2..8ab1562 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -58,6 +58,7 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag); - int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); +@@ -437,7 +433,7 @@ class CLI(object): -+int loc_network_cmp(struct loc_network* self, struct loc_network* other); - int loc_network_eq(struct loc_network* self, struct loc_network* other); - int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); -diff --git a/src/network-list.c b/src/network-list.c -index b455caf..6e9cd37 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -16,6 +16,7 @@ + # Try downloading a new database + try: +- t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir) ++ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir) - #include - #include -+#include + # If no file could be downloaded, log a message + except FileNotFoundError as e: +@@ -453,13 +449,7 @@ class CLI(object): - #include - #include -@@ -130,11 +131,71 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis - return loc_network_ref(list->elements[index]); - } + return 0 -+//MOVE FUNCTION GOES HERE -+ -+static off_t loc_network_list_find(struct loc_network_list* list, -+ struct loc_network* network, int* found) { -+ off_t lo = 0; -+ off_t hi = list->size - 1; -+ -+ *found = 0; -+ -+#ifdef ENABLE_DEBUG -+ // Save start time -+ clock_t start = clock(); -+#endif -+ -+ off_t i = 0; -+ -+ while (lo <= hi) { -+ i = (lo + hi) / 2; -+ -+ // Check if this is a match -+ int result = loc_network_cmp(network, list->elements[i]); -+ -+ if (result == 0) { -+ *found = 1; -+ -+#ifdef ENABLE_DEBUG -+ clock_t end = clock(); +- def handle_verify(self, ns): +- try: +- db = location.Database(ns.database) +- except FileNotFoundError as e: +- log.error("%s: %s" % (ns.database, e)) +- return 127 +- ++ def handle_verify(self, db, ns): + # Verify the database + with open(ns.public_key, "r") as f: + if not db.verify(f): +diff --git a/src/python/network.c b/src/python/network.c +index 5496d1e..b6e92fb 100644 +--- a/src/python/network.c ++++ b/src/python/network.c +@@ -20,10 +20,29 @@ + + #include + #include ++#include + + #include "locationmodule.h" + #include "network.h" + ++static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) { ++ PyObject* list = PyList_New(0); ++ if (!networks) ++ return list; + -+ // Log how fast this has been -+ DEBUG(list->ctx, "Found network in %.4fms at %jd\n", -+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); -+#endif ++ while (!loc_network_list_empty(networks)) { ++ struct loc_network* network = loc_network_list_pop(networks); + -+ return i; -+ } ++ PyObject* n = new_network(&NetworkType, network); ++ PyList_Append(list, n); + -+ if (result > 0) -+ lo = i + 1; -+ else -+ hi = i - 1; ++ loc_network_unref(network); ++ Py_DECREF(n); + } + -+#ifdef ENABLE_DEBUG -+ clock_t end = clock(); ++ return list; ++} + -+ // Log how fast this has been -+ DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n", -+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); -+#endif + PyObject* new_network(PyTypeObject* type, struct loc_network* network) { + NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0); + if (self) { +@@ -154,13 +173,28 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) { + Py_RETURN_NONE; + } + ++static PyObject* Network_exclude(NetworkObject* self, PyObject* args) { ++ NetworkObject* other = NULL; + -+ return i; -+} ++ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) ++ return NULL; + - LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -- // Do not add networks that are already on the list -- if (loc_network_list_contains(list, network)) -+ int found = 0; ++ struct loc_network_list* list = loc_network_exclude(self->network, other->network); + -+ off_t index = loc_network_list_find(list, network, &found); ++ // Convert to Python objects ++ PyObject* obj = PyList_FromNetworkList(list); ++ loc_network_list_unref(list); + -+ // The network has been found on the list. Nothing to do. -+ if (found) - return 0; - -+ DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n", -+ list, network, (intmax_t)index); ++ return obj; ++} + - // Check if we have space left - if (list->size >= list->elements_size) { - int r = loc_network_list_grow(list, 64); -@@ -142,9 +203,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n - return r; - } + static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { + NetworkObject* other = NULL; -- DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); -+ // The list is now larger -+ list->size++; -+ -+ // Move all elements out of the way -+ for (unsigned int i = list->size - 1; i > index; i--) -+ list->elements[i] = list->elements[i - 1]; + if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) + return NULL; -- list->elements[list->size++] = loc_network_ref(network); -+ // Add the new element at the right place -+ list->elements[index] = loc_network_ref(network); +- if (loc_network_is_subnet_of(self->network, other->network)) ++ if (loc_network_is_subnet(other->network, self->network)) + Py_RETURN_TRUE; - return 0; - } -@@ -183,12 +250,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis + Py_RETURN_FALSE; +@@ -181,6 +215,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) { + return obj; } - LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { -- for (unsigned int i = 0; i < list->size; i++) { -- if (loc_network_eq(list->elements[i], network)) -- return 1; -- } -+ int found = 0; - -- return 0; -+ loc_network_list_find(list, network, &found); ++static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) { ++ struct in_addr address4; + -+ return found; - } - - static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { -diff --git a/src/network.c b/src/network.c -index 72b77e6..38d557a 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -441,6 +441,28 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag - return loc_network_has_flag(network, flag); - } - -+LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) { -+ // Compare family -+ if (self->family > other->family) -+ return 1; -+ else if (self->family < other->family) -+ return -1; ++ // Convert IPv4 addresses to struct in_addr ++ if (IN6_IS_ADDR_V4MAPPED(address6)) { ++ address4.s_addr = address6->s6_addr32[3]; + -+ // Compare address -+ int r = in6_addr_cmp(&self->first_address, &other->first_address); -+ if (r) -+ return r; ++ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4)); ++ } + -+ // Compare prefix -+ if (self->prefix > other->prefix) -+ return 1; -+ else if (self->prefix < other->prefix) -+ return -1; ++ // Return IPv6 addresses as they are ++ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6)); ++} + -+ // Both networks are equal -+ return 0; ++static PyObject* Network_get__first_address(NetworkObject* self) { ++ const struct in6_addr* address = loc_network_get_first_address(self->network); ++ ++ return PyBytes_FromAddress(address); +} + - LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { - // Family must be the same - if (self->family != other->family) --- -2.20.1 - -From 39cbbc63aee362d82f69a9b4722b59153ce799a0 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 24 Nov 2020 15:04:03 +0000 -Subject: [PATCH 088/111] network-list: Drop sorting functions - -Since the list is always sorted, there is no point in -sorting it again... - -Signed-off-by: Michael Tremer ---- - src/database.c | 6 +---- - src/libloc.sym | 3 --- - src/loc/network-list.h | 3 --- - src/network-list.c | 52 ------------------------------------------ - src/test-network.c | 9 -------- - 5 files changed, 1 insertion(+), 72 deletions(-) - -diff --git a/src/database.c b/src/database.c -index ba4f98a..5546091 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1372,17 +1372,13 @@ static int __loc_database_enumerator_next_network_flattened( - // We no longer need the excluded list - loc_network_list_unref(excluded); - -- // Sort all subnets -- loc_network_list_sort(subnets); -- - // Replace network with the first one - loc_network_unref(*network); - - *network = loc_network_list_pop_first(subnets); - - // Push the rest onto the stack -- loc_network_list_merge_reverse(enumerator->stack, subnets); -- -+ loc_network_list_merge(enumerator->stack, subnets); - loc_network_list_unref(subnets); - - return 0; -diff --git a/src/libloc.sym b/src/libloc.sym -index 28cc8e8..4b0ce45 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -141,15 +141,12 @@ global: - loc_network_list_empty; - loc_network_list_get; - loc_network_list_merge; -- loc_network_list_merge_reverse; - loc_network_list_new; - loc_network_list_pop; - loc_network_list_pop_first; - loc_network_list_push; - loc_network_list_ref; -- loc_network_list_reverse; - loc_network_list_size; -- loc_network_list_sort; - loc_network_list_unref; - - # Writer -diff --git a/src/loc/network-list.h b/src/loc/network-list.h -index 89776a6..21c7402 100644 ---- a/src/loc/network-list.h -+++ b/src/loc/network-list.h -@@ -30,9 +30,6 @@ int loc_network_list_push(struct loc_network_list* list, struct loc_network* net - struct loc_network* loc_network_list_pop(struct loc_network_list* list); - struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); - int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); --void loc_network_list_sort(struct loc_network_list* list); --void loc_network_list_reverse(struct loc_network_list* list); - int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); --int loc_network_list_merge_reverse(struct loc_network_list* self, struct loc_network_list* other); - - #endif -diff --git a/src/network-list.c b/src/network-list.c -index 6e9cd37..7e8b5f3 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -257,45 +257,6 @@ LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct l - return found; - } - --static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { -- // Do nothing for invalid indices -- if (i1 >= list->size || i2 >= list->size) -- return; -- -- struct loc_network* network1 = list->elements[i1]; -- struct loc_network* network2 = list->elements[i2]; -- -- list->elements[i1] = network2; -- list->elements[i2] = network1; --} -- --LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { -- unsigned int i = 0; -- unsigned int j = list->size - 1; -- -- while (i < j) { -- loc_network_list_swap(list, i++, j--); -- } --} -- --LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { -- unsigned int n = list->size; -- int swapped; -- -- do { -- swapped = 0; -- -- for (unsigned int i = 1; i < n; i++) { -- if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) { -- loc_network_list_swap(list, i-1, i); -- swapped = 1; -- } -- } -- -- n--; -- } while (swapped); --} -- - LOC_EXPORT int loc_network_list_merge( - struct loc_network_list* self, struct loc_network_list* other) { - int r; -@@ -308,16 +269,3 @@ LOC_EXPORT int loc_network_list_merge( + static PyObject* Network_get_last_address(NetworkObject* self) { + char* address = loc_network_format_last_address(self->network); - return 0; +@@ -190,7 +244,19 @@ static PyObject* Network_get_last_address(NetworkObject* self) { + return obj; } -- --LOC_EXPORT int loc_network_list_merge_reverse( -- struct loc_network_list* self, struct loc_network_list* other) { -- int r; -- -- for (int i = other->size - 1; i >= 0; i--) { -- r = loc_network_list_push(self, other->elements[i]); -- if (r) -- return r; -- } -- -- return 0; --} -diff --git a/src/test-network.c b/src/test-network.c -index 79c2967..8a6763c 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -188,15 +188,6 @@ int main(int argc, char** argv) { - } - - loc_network_list_dump(excluded); -- -- // Reverse it -- loc_network_list_reverse(excluded); -- loc_network_list_dump(excluded); -- -- // Sort them and dump them again -- loc_network_list_sort(excluded); -- loc_network_list_dump(excluded); -- - loc_network_list_unref(excluded); - - // Create a database --- -2.20.1 - -From 5dacb45afceac2d05ea597755c1ca5a1b62cc0fd Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 24 Nov 2020 15:15:59 +0000 -Subject: [PATCH 089/111] database: Avoid merging the same data twice - -When finish splitting networks into many parts, we have -a list of subnets with the excluded subnets and merge them -together first and put them on the stack again. - -This is slower than pushing it all onto the stack first -and then popping the first element. - -Signed-off-by: Michael Tremer ---- - src/database.c | 24 ++++++++++++++---------- - 1 file changed, 14 insertions(+), 10 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 5546091..f1f6ae0 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1360,8 +1360,8 @@ static int __loc_database_enumerator_next_network_flattened( - return -1; - } - -- // Merge excluded list with subnets -- r = loc_network_list_merge(subnets, excluded); -+ // Merge subnets onto the stack -+ r = loc_network_list_merge(enumerator->stack, subnets); - if (r) { - loc_network_list_unref(subnets); - loc_network_list_unref(excluded); -@@ -1369,17 +1369,21 @@ static int __loc_database_enumerator_next_network_flattened( - return r; - } - -- // We no longer need the excluded list -- loc_network_list_unref(excluded); -- -- // Replace network with the first one -- loc_network_unref(*network); -+ // Push excluded list onto the stack -+ r = loc_network_list_merge(enumerator->stack, excluded); -+ if (r) { -+ loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); - -- *network = loc_network_list_pop_first(subnets); -+ return r; -+ } -- // Push the rest onto the stack -- loc_network_list_merge(enumerator->stack, subnets); - loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); ++static PyObject* Network_get__last_address(NetworkObject* self) { ++ const struct in6_addr* address = loc_network_get_last_address(self->network); + -+ // Replace network with the first one from the stack -+ loc_network_unref(*network); -+ *network = loc_network_list_pop_first(enumerator->stack); - - return 0; - } --- -2.20.1 - -From 04cbd2bfa892fc7374ad506ec7ba81727c5a4b96 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 24 Nov 2020 15:22:02 +0000 -Subject: [PATCH 090/111] database: Read the first element from the list - -Signed-off-by: Michael Tremer ---- - src/database.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/database.c b/src/database.c -index f1f6ae0..914ed3e 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1191,7 +1191,7 @@ static int __loc_database_enumerator_next_network( - struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { - // Return top element from the stack - while (1) { -- *network = loc_network_list_pop(enumerator->stack); -+ *network = loc_network_list_pop_first(enumerator->stack); - - // Stack is empty - if (!*network) --- -2.20.1 - -From c650008e2dd9af5fd1eadba6354aa0b615047f84 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 24 Nov 2020 15:41:53 +0000 -Subject: [PATCH 091/111] network: Add excluded networks to to_check list - -This is now being done immediately instead of creating a new -list which is being merged later. - -Signed-off-by: Michael Tremer ---- - src/network.c | 55 ++++++++++++++++++++++++++++++--------------------- - 1 file changed, 32 insertions(+), 23 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 38d557a..7ab22f8 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -640,49 +640,55 @@ ERROR: - return r; - } - --LOC_EXPORT struct loc_network_list* loc_network_exclude( -- struct loc_network* self, struct loc_network* other) { -- struct loc_network_list* list; -- --#ifdef ENABLE_DEBUG -- char* n1 = loc_network_str(self); -- char* n2 = loc_network_str(other); -- -- DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); -- -- free(n1); -- free(n2); --#endif -- -+static int __loc_network_exclude_to_list(struct loc_network* self, -+ struct loc_network* other, struct loc_network_list* list) { - // Family must match - if (self->family != other->family) { - DEBUG(self->ctx, "Family mismatch\n"); - -- return NULL; -+ return 1; - } - - // Other must be a subnet of self - if (!loc_network_is_subnet(self, other)) { - DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); ++ return PyBytes_FromAddress(address); ++} ++ + static struct PyMethodDef Network_methods[] = { ++ { ++ "exclude", ++ (PyCFunction)Network_exclude, ++ METH_VARARGS, ++ NULL, ++ }, + { + "has_flag", + (PyCFunction)Network_has_flag, +@@ -241,6 +307,13 @@ static struct PyGetSetDef Network_getsetters[] = { + NULL, + NULL, + }, ++ { ++ "_first_address", ++ (getter)Network_get__first_address, ++ NULL, ++ NULL, ++ NULL, ++ }, + { + "last_address", + (getter)Network_get_last_address, +@@ -248,6 +321,13 @@ static struct PyGetSetDef Network_getsetters[] = { + NULL, + NULL, + }, ++ { ++ "_last_address", ++ (getter)Network_get__last_address, ++ NULL, ++ NULL, ++ NULL, ++ }, + { NULL }, + }; -- return NULL; -+ return 1; - } +diff --git a/src/test-as.c b/src/test-as.c +index 839a04c..2d61675 100644 +--- a/src/test-as.c ++++ b/src/test-as.c +@@ -95,7 +95,7 @@ int main(int argc, char** argv) { + // Enumerator - // We cannot perform this operation if both networks equal - if (loc_network_eq(self, other)) { - DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); + struct loc_database_enumerator* enumerator; +- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES); ++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0); + if (err) { + fprintf(stderr, "Could not create a database enumerator\n"); + exit(EXIT_FAILURE); +diff --git a/src/test-database.c b/src/test-database.c +index b4a75c4..da4b11c 100644 +--- a/src/test-database.c ++++ b/src/test-database.c +@@ -38,6 +38,14 @@ const char* DESCRIPTION = + "Maecenas ut venenatis nunc."; + const char* LICENSE = "CC"; -- return NULL; -+ return 1; ++const char* networks[] = { ++ "2001:db8::/32", ++ "2001:db8:1000::/48", ++ "2001:db8:2000::/48", ++ "2001:db8:2020::/48", ++ NULL, ++}; ++ + static int attempt_to_open(struct loc_ctx* ctx, char* path) { + FILE* f = fopen(path, "r"); + if (!f) +@@ -139,6 +147,24 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); } -+ return __loc_network_exclude(self, other, list); -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_exclude( -+ struct loc_network* self, struct loc_network* other) { -+ struct loc_network_list* list; -+ -+#ifdef ENABLE_DEBUG -+ char* n1 = loc_network_str(self); -+ char* n2 = loc_network_str(other); ++ struct loc_network* network = NULL; + -+ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); ++ // Add some networks ++ const char** n = networks; ++ while (*n) { ++ err = loc_writer_add_network(writer, &network, *n); ++ if (err) { ++ fprintf(stderr, "Could not add network %s\n", *n); ++ exit(EXIT_FAILURE); ++ } + -+ free(n1); -+ free(n2); -+#endif ++ // Set a country ++ loc_network_set_country_code(network, "XX"); + - // Create a new list with the result - int r = loc_network_list_new(self->ctx, &list); - if (r) { - ERROR(self->ctx, "Could not create network list: %d\n", r); ++ // Next one ++ n++; ++ } + - return NULL; + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +@@ -170,6 +196,33 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); } -- r = __loc_network_exclude(self, other, list); -+ r = __loc_network_exclude_to_list(self, other, list); - if (r) { - loc_network_list_unref(list); - -@@ -709,11 +715,14 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - subnet = loc_network_list_get(list, i); - - // Find all excluded networks -- struct loc_network_list* excluded = loc_network_exclude(network, subnet); -- if (excluded) { -- // Add them all to the "to check" list -- loc_network_list_merge(to_check, excluded); -- loc_network_list_unref(excluded); -+ if (!loc_network_list_contains(to_check, subnet)) { -+ r = __loc_network_exclude_to_list(network, subnet, to_check); -+ if (r) { -+ loc_network_list_unref(to_check); -+ loc_network_unref(subnet); ++ // Enumerator ++ struct loc_database_enumerator* enumerator; ++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0); ++ if (err) { ++ fprintf(stderr, "Could not initialise the enumerator: %d\n", err); ++ exit(EXIT_FAILURE); ++ } + -+ return NULL; -+ } - } - - // Cleanup --- -2.20.1 - -From 7c6983ad52724d395446bdbd24d36b2ce22aecfd Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 24 Nov 2020 15:42:54 +0000 -Subject: [PATCH 092/111] test: Add more networks to list to see the algorithm - work - -Signed-off-by: Michael Tremer ---- - src/test-network-list.c | 32 +++++++++++++++++++++++++++++++- - 1 file changed, 31 insertions(+), 1 deletion(-) - ++ // Walk through all networks ++ while (1) { ++ err = loc_database_enumerator_next_network(enumerator, &network); ++ if (err) { ++ fprintf(stderr, "Error fetching the next network: %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (!network) ++ break; ++ ++ char* s = loc_network_str(network); ++ printf("Got network: %s\n", s); ++ free(s); ++ } ++ ++ // Free the enumerator ++ loc_database_enumerator_unref(enumerator); ++ + // Close the database + loc_database_unref(db); + loc_unref(ctx); diff --git a/src/test-network-list.c b/src/test-network-list.c -index 3061d63..8253fc7 100644 ---- a/src/test-network-list.c +new file mode 100644 +index 0000000..6f32ff7 +--- /dev/null +++ b/src/test-network-list.c -@@ -58,6 +58,20 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - +@@ -0,0 +1,183 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2017 IPFire Development Team ++ ++ This program is free software; you can redistribute it and/or modify ++ it under the terms of the GNU General Public License as published by ++ the Free Software Foundation; either version 2 of the License, or ++ (at your option) any later version. ++ ++ This program is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ GNU General Public License for more details. ++*/ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++ ++int main(int argc, char** argv) { ++ int err; ++ ++ struct loc_ctx* ctx; ++ err = loc_new(&ctx); ++ if (err < 0) ++ exit(EXIT_FAILURE); ++ ++ // Enable debug logging ++ loc_set_log_priority(ctx, LOG_DEBUG); ++ ++ // Create a network ++ struct loc_network* network1; ++ err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32"); ++ if (err) { ++ fprintf(stderr, "Could not create the network1\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ struct loc_network* subnet1; ++ err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48"); ++ if (err) { ++ fprintf(stderr, "Could not create the subnet1\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ struct loc_network* subnet2; ++ err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48"); ++ if (err) { ++ fprintf(stderr, "Could not create the subnet2\n"); ++ exit(EXIT_FAILURE); ++ } ++ + struct loc_network* subnet3; + err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48"); + if (err) { @@ -9786,325 +3434,117 @@ index 3061d63..8253fc7 100644 + exit(EXIT_FAILURE); + } + - // Make a list with both subnets - struct loc_network_list* subnets; - err = loc_network_list_new(ctx, &subnets); -@@ -89,8 +103,24 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ // Add the fourth one next -+ err = loc_network_list_push(subnets, subnet4); -+ if (err) { -+ fprintf(stderr, "Could not add subnet4 to subnets list\n"); ++ struct loc_network* subnet5; ++ err = loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48"); ++ if (err) { ++ fprintf(stderr, "Could not create the subnet5\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ struct loc_network* subnet6; ++ err = loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48"); ++ if (err) { ++ fprintf(stderr, "Could not create the subnet6\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Make a list with both subnets ++ struct loc_network_list* subnets; ++ err = loc_network_list_new(ctx, &subnets); ++ if (err) { ++ fprintf(stderr, "Could not create subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ size_t size = loc_network_list_size(subnets); ++ if (size > 0) { ++ fprintf(stderr, "The list is not empty: %zu\n", size); ++ exit(EXIT_FAILURE); ++ } ++ ++ err = loc_network_list_push(subnets, subnet1); ++ if (err) { ++ fprintf(stderr, "Could not add subnet1 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (loc_network_list_empty(subnets)) { ++ fprintf(stderr, "The subnets list reports that it is empty\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ err = loc_network_list_push(subnets, subnet2); ++ if (err) { ++ fprintf(stderr, "Could not add subnet2 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Add the fourth one next ++ err = loc_network_list_push(subnets, subnet4); ++ if (err) { ++ fprintf(stderr, "Could not add subnet4 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Add the third one ++ err = loc_network_list_push(subnets, subnet3); ++ if (err) { ++ fprintf(stderr, "Could not add subnet3 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Add more subnets ++ err = loc_network_list_push(subnets, subnet5); ++ if (err) { ++ fprintf(stderr, "Could not add subnet5 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ err = loc_network_list_push(subnets, subnet6); ++ if (err) { ++ fprintf(stderr, "Could not add subnet6 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ loc_network_list_dump(subnets); ++ ++ size = loc_network_list_size(subnets); ++ if (size != 6) { ++ fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Exclude subnet1 from network1 ++ struct loc_network_list* excluded = loc_network_exclude(network1, subnet1); ++ if (!excluded) { ++ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n"); + exit(EXIT_FAILURE); + } + -+ // Add the third one -+ err = loc_network_list_push(subnets, subnet3); -+ if (err) { -+ fprintf(stderr, "Could not add subnet3 to subnets list\n"); ++ loc_network_list_dump(excluded); ++ ++ // Exclude all subnets from network1 ++ excluded = loc_network_exclude_list(network1, subnets); ++ if (!excluded) { ++ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n"); + exit(EXIT_FAILURE); + } + -+ loc_network_list_dump(subnets); ++ loc_network_list_dump(excluded); + - size = loc_network_list_size(subnets); -- if (size != 2) { -+ if (size != 4) { - fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size); - exit(EXIT_FAILURE); - } --- -2.20.1 - -From da101d55e66ebaeeed8b0b16829a2022a1af0678 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 24 Nov 2020 15:48:55 +0000 -Subject: [PATCH 093/111] Drop loc_network_eq in favour of loc_network_cmp - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 1 - - src/loc/network.h | 1 - - src/network.c | 24 ++++-------------------- - src/test-network.c | 8 ++++---- - 4 files changed, 8 insertions(+), 26 deletions(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 4b0ce45..d8e8f14 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -107,7 +107,6 @@ global: - # Network - loc_network_address_family; - loc_network_cmp; -- loc_network_eq; - loc_network_exclude; - loc_network_exclude_list; - loc_network_format_first_address; -diff --git a/src/loc/network.h b/src/loc/network.h -index 8ab1562..d5d0ccd 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -59,7 +59,6 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_cmp(struct loc_network* self, struct loc_network* other); --int loc_network_eq(struct loc_network* self, struct loc_network* other); - int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); -diff --git a/src/network.c b/src/network.c -index 7ab22f8..503bf3d 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -463,22 +463,6 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth - return 0; - } - --LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { -- // Family must be the same -- if (self->family != other->family) -- return 0; -- -- // The start address must be the same -- if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) -- return 0; -- -- // The prefix length must be the same -- if (self->prefix != other->prefix) -- return 0; -- -- return 1; --} -- - LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) { - // Families must match - if (self->family != other->family) -@@ -596,12 +580,12 @@ static int __loc_network_exclude(struct loc_network* network, - if (r) - goto ERROR; - -- if (loc_network_eq(other, subnet1)) { -+ if (loc_network_cmp(other, subnet1) == 0) { - r = loc_network_list_push(list, subnet2); - if (r) - goto ERROR; - -- } else if (loc_network_eq(other, subnet2)) { -+ } else if (loc_network_cmp(other, subnet2) == 0) { - r = loc_network_list_push(list, subnet1); - if (r) - goto ERROR; -@@ -657,7 +641,7 @@ static int __loc_network_exclude_to_list(struct loc_network* self, - } - - // We cannot perform this operation if both networks equal -- if (loc_network_eq(self, other)) { -+ if (loc_network_cmp(self, other) == 0) { - DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); - - return 1; -@@ -745,7 +729,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - subnet = loc_network_list_get(list, i); - - // Drop this subnet if is is already in list -- if (loc_network_eq(subnet_to_check, subnet)) { -+ if (loc_network_cmp(subnet_to_check, subnet) == 0) { - passed = 0; - loc_network_unref(subnet); - break; -diff --git a/src/test-network.c b/src/test-network.c -index 8a6763c..f4cf97b 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -125,14 +125,14 @@ int main(int argc, char** argv) { - #endif - - // Check equals function -- err = loc_network_eq(network1, network1); -- if (!err) { -+ err = loc_network_cmp(network1, network1); -+ if (err) { - fprintf(stderr, "Network is not equal with itself\n"); - exit(EXIT_FAILURE); - } - -- err = loc_network_eq(network1, network2); -- if (err) { -+ err = loc_network_cmp(network1, network2); -+ if (!err) { - fprintf(stderr, "Networks equal unexpectedly\n"); - exit(EXIT_FAILURE); - } --- -2.20.1 - -From 61a6f6e4bf4493236d796e94f3d721bcf5ba68aa Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 24 Nov 2020 15:50:39 +0000 -Subject: [PATCH 094/111] Drop loc_network_gt which is now unused - -Signed-off-by: Michael Tremer ---- - src/libloc.sym | 1 - - src/loc/network.h | 1 - - src/network.c | 27 --------------------------- - 3 files changed, 29 deletions(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index d8e8f14..cb5e8ef 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -115,7 +115,6 @@ global: - loc_network_get_country_code; - loc_network_get_first_address; - loc_network_get_last_address; -- loc_network_gt; - loc_network_has_flag; - loc_network_is_subnet; - loc_network_match_asn; -diff --git a/src/loc/network.h b/src/loc/network.h -index d5d0ccd..af3dafd 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -59,7 +59,6 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_cmp(struct loc_network* self, struct loc_network* other); --int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); - int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2); -diff --git a/src/network.c b/src/network.c -index 503bf3d..ac478d5 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -463,33 +463,6 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth - return 0; - } - --LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) { -- // Families must match -- if (self->family != other->family) -- return -1; -- -- int r = in6_addr_cmp(&self->first_address, &other->first_address); -- -- switch (r) { -- // Smaller -- case -1: -- return 0; -- -- // Larger -- case 1: -- return 1; -- -- default: -- break; -- } -- -- if (self->prefix > other->prefix) -- return 1; -- -- // Dunno -- return 0; --} -- - LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { - if (loc_network_match_address(self, &other->first_address) == 0) - return 1; --- -2.20.1 - -From fc692a58d9f4ca958f88cfa202250c572a0af6ea Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 24 Nov 2020 16:50:17 +0000 -Subject: [PATCH 095/111] network: Adjust return codes of - loc_network_match_address and add test - -Signed-off-by: Michael Tremer ---- - src/database.c | 3 +-- - src/libloc.sym | 1 + - src/network.c | 16 +++++++++------- - src/test-network.c | 14 ++++++++++++++ - 4 files changed, 25 insertions(+), 9 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 914ed3e..1871b74 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -776,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru - } - - // Check if the given IP address is inside the network -- r = loc_network_match_address(*network, address); -- if (r) { -+ if (!loc_network_match_address(*network, address)) { - DEBUG(db->ctx, "Searched address is not part of the network\n"); - - loc_network_unref(*network); -diff --git a/src/libloc.sym b/src/libloc.sym -index cb5e8ef..ee333f1 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -117,6 +117,7 @@ global: - loc_network_get_last_address; - loc_network_has_flag; - loc_network_is_subnet; -+ loc_network_match_address; - loc_network_match_asn; - loc_network_match_country_code; - loc_network_match_flag; -diff --git a/src/network.c b/src/network.c -index ac478d5..febab95 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -374,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) { - LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) { - // Address must be larger than the start address - if (in6_addr_cmp(&network->first_address, address) > 0) -- return 1; -+ return 0; - - // Address must be smaller than the last address - if (in6_addr_cmp(&network->last_address, address) < 0) -- return 1; -+ return 0; - - // The address is inside this network -- return 0; -+ return 1; - } - - LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) { -@@ -464,16 +464,18 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth - } - - LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { -- if (loc_network_match_address(self, &other->first_address) == 0) -+ // Either of the start addresses must be in the other subnet -+ if (loc_network_match_address(self, &other->first_address)) - return 1; - -- if (loc_network_match_address(self, &other->last_address) == 0) -+ if (loc_network_match_address(other, &self->first_address)) - return 1; - -- if (loc_network_match_address(other, &self->first_address) == 0) -+ // Or either of the end addresses is in the other subnet -+ if (loc_network_match_address(self, &other->last_address)) - return 1; - -- if (loc_network_match_address(other, &self->last_address) == 0) -+ if (loc_network_match_address(other, &self->last_address)) - return 1; - - return 0; ++ if (excluded) ++ loc_network_list_unref(excluded); ++ ++ loc_network_list_unref(subnets); ++ loc_network_unref(network1); ++ loc_network_unref(subnet1); ++ loc_network_unref(subnet2); ++ loc_unref(ctx); ++ ++ return EXIT_SUCCESS; ++} diff --git a/src/test-network.c b/src/test-network.c -index f4cf97b..339743d 100644 +index d38f13d..dde13f1 100644 --- a/src/test-network.c +++ b/src/test-network.c @@ -14,6 +14,7 @@ @@ -10115,280 +3555,128 @@ index f4cf97b..339743d 100644 #include #include #include -@@ -46,6 +47,13 @@ int main(int argc, char** argv) { - } - #endif +@@ -37,12 +38,21 @@ int main(int argc, char** argv) { + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); ++#if 0 + struct loc_network_tree* tree; + err = loc_network_tree_new(ctx, &tree); + if (err) { + fprintf(stderr, "Could not create the network tree\n"); + exit(EXIT_FAILURE); + } ++#endif ++ + struct in6_addr address; + err = inet_pton(AF_INET6, "2001:db8::1", &address); + if (err != 1) { + fprintf(stderr, "Could not parse IP address\n"); + exit(EXIT_FAILURE); + } -+ + // Create a network struct loc_network* network1; - err = loc_network_new_from_string(ctx, &network1, "2001:db8::1/32"); -@@ -92,6 +100,12 @@ int main(int argc, char** argv) { +@@ -58,12 +68,14 @@ int main(int argc, char** argv) { exit(EXIT_FAILURE); } -+ err = loc_network_match_address(network1, &address); -+ if (!err) { -+ fprintf(stderr, "Network1 does not match address\n"); -+ exit(EXIT_FAILURE); -+ } -+ - struct loc_network* network2; - err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48"); ++#if 0 + // Adding network to the tree + err = loc_network_tree_add_network(tree, network1); if (err) { --- -2.20.1 - -From 06177d8c6004bf8b54322d92926152a7656f9c2a Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 24 Nov 2020 16:57:28 +0000 -Subject: [PATCH 096/111] network-list: Use binary search to find if a network - is a subnet - -Signed-off-by: Michael Tremer ---- - src/network.c | 13 ++++++------- - 1 file changed, 6 insertions(+), 7 deletions(-) - -diff --git a/src/network.c b/src/network.c -index febab95..394bafc 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -697,19 +697,18 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - while (!loc_network_list_empty(to_check)) { - struct loc_network* subnet_to_check = loc_network_list_pop(to_check); - -+ // Check whether the subnet to check is part of the input list -+ if (loc_network_list_contains(list, subnet_to_check)) { -+ loc_network_unref(subnet_to_check); -+ continue; -+ } -+ - // Marks whether this subnet passed all checks - int passed = 1; - - for (unsigned int i = 0; i < loc_network_list_size(list); i++) { - subnet = loc_network_list_get(list, i); - -- // Drop this subnet if is is already in list -- if (loc_network_cmp(subnet_to_check, subnet) == 0) { -- passed = 0; -- loc_network_unref(subnet); -- break; -- } -- - // Drop this subnet if is a subnet of another subnet - if (loc_network_is_subnet(subnet_to_check, subnet)) { - passed = 0; --- -2.20.1 - -From 77e6d5379ea0de3264be5b8918d620d16e6bcbd3 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Tue, 24 Nov 2020 19:39:35 +0000 -Subject: [PATCH 097/111] network-list: Check last element before doing binary - search - -This is helpful because very often we walk through a list in -order and are most interested in the last element. - -Signed-off-by: Michael Tremer ---- - src/network-list.c | 24 ++++++++++++++++++++++-- - 1 file changed, 22 insertions(+), 2 deletions(-) - -diff --git a/src/network-list.c b/src/network-list.c -index 7e8b5f3..c86ad07 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -137,8 +137,26 @@ static off_t loc_network_list_find(struct loc_network_list* list, - struct loc_network* network, int* found) { - off_t lo = 0; - off_t hi = list->size - 1; -+ int result; - -- *found = 0; -+ // Since we are working on an ordered list, there is often a good chance that -+ // the network we are looking for is at the end or has to go to the end. -+ if (hi >= 0) { -+ result = loc_network_cmp(network, list->elements[hi]); -+ -+ // Match, so we are done -+ if (result == 0) { -+ *found = 1; -+ -+ return hi; -+ -+ // This needs to be added after the last one -+ } else if (result > 0) { -+ *found = 0; -+ -+ return hi + 1; -+ } -+ } - - #ifdef ENABLE_DEBUG - // Save start time -@@ -151,7 +169,7 @@ static off_t loc_network_list_find(struct loc_network_list* list, - i = (lo + hi) / 2; - - // Check if this is a match -- int result = loc_network_cmp(network, list->elements[i]); -+ result = loc_network_cmp(network, list->elements[i]); - - if (result == 0) { - *found = 1; -@@ -173,6 +191,8 @@ static off_t loc_network_list_find(struct loc_network_list* list, - hi = i - 1; + fprintf(stderr, "Could not add network to the tree\n"); + exit(EXIT_FAILURE); } ++#endif -+ *found = 0; -+ - #ifdef ENABLE_DEBUG - clock_t end = clock(); - --- -2.20.1 - -From d42f34ce9b0aa2382ae9b852882c62b34d367cfe Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 14:41:39 +0000 -Subject: [PATCH 098/111] network-list: Set elements pointer to NULL so that we - know it is empty - -Signed-off-by: Michael Tremer ---- - src/network-list.c | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/network-list.c b/src/network-list.c -index c86ad07..2c4edb3 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -104,6 +104,7 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { - loc_network_unref(list->elements[i]); - - free(list->elements); -+ list->elements = NULL; - list->elements_size = 0; - - list->size = 0; --- -2.20.1 - -From 673e03f7bfc807248a769cb00ec80f0fa2af7a25 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 14:42:26 +0000 -Subject: [PATCH 099/111] network-list: Do not half list when popping the first - element - -The list was unfortunately halved in size every time an element -was taken from it, which was great for performance, but shortened -the result substantially. - -Signed-off-by: Michael Tremer ---- - src/network-list.c | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/src/network-list.c b/src/network-list.c -index 2c4edb3..f4a9d05 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -261,10 +261,13 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis - struct loc_network* network = list->elements[0]; - - // Move all elements to the top of the stack -- for (unsigned int i = 0; i < --list->size; i++) { -+ for (unsigned int i = 0; i < list->size - 1; i++) { - list->elements[i] = list->elements[i+1]; + // Check if the first and last addresses are correct + char* string = loc_network_format_first_address(network1); +@@ -88,6 +100,12 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); } -+ // The list is shorter now -+ --list->size; ++ err = loc_network_match_address(network1, &address); ++ if (!err) { ++ fprintf(stderr, "Network1 does not match address\n"); ++ exit(EXIT_FAILURE); ++ } + - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); - - return network; --- -2.20.1 - -From 9446c7538ab8b27486d74f6dc0dac8bb5c1bbe92 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 14:43:58 +0000 -Subject: [PATCH 100/111] network-list: Show index when listing networks - -Signed-off-by: Michael Tremer ---- - src/network-list.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/network-list.c b/src/network-list.c -index f4a9d05..cf9459d 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -119,7 +119,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { + struct loc_network* network2; + err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48"); + if (err) { +@@ -101,6 +119,7 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } - s = loc_network_str(network); ++#if 0 + // Adding network to the tree + err = loc_network_tree_add_network(tree, network2); + if (err) { +@@ -117,20 +136,84 @@ int main(int argc, char** argv) { -- INFO(list->ctx, "%s\n", s); -+ INFO(list->ctx, "%4d: %s\n", i, s); - free(s); + size_t nodes = loc_network_tree_count_nodes(tree); + printf("The tree has %zu nodes\n", nodes); ++#endif ++ ++ // Check equals function ++ err = loc_network_cmp(network1, network1); ++ if (err) { ++ fprintf(stderr, "Network is not equal with itself\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ err = loc_network_cmp(network1, network2); ++ if (!err) { ++ fprintf(stderr, "Networks equal unexpectedly\n"); ++ exit(EXIT_FAILURE); ++ } + + // Check subnet function +- err = loc_network_is_subnet_of(network1, network2); +- if (err != 0) { ++ err = loc_network_is_subnet(network1, network2); ++ if (!err) { + fprintf(stderr, "Subnet check 1 failed: %d\n", err); + exit(EXIT_FAILURE); } - } --- -2.20.1 - -From dc31666be416dbb3bd2bc04127104a3f75b42d5c Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 14:44:23 +0000 -Subject: [PATCH 101/111] network-list: Remove useless comment - -Signed-off-by: Michael Tremer ---- - src/network-list.c | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/src/network-list.c b/src/network-list.c -index cf9459d..bd3d64e 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -132,8 +132,6 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis - return loc_network_ref(list->elements[index]); - } --//MOVE FUNCTION GOES HERE -- - static off_t loc_network_list_find(struct loc_network_list* list, - struct loc_network* network, int* found) { - off_t lo = 0; --- -2.20.1 - -From 82fa4c92c88cc172953198637bccc375c4d25d20 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 14:44:56 +0000 -Subject: [PATCH 102/111] networks: Add tests for overlaps function - -Signed-off-by: Michael Tremer ---- - src/test-network.c | 10 ++++++++++ - 1 file changed, 10 insertions(+) - -diff --git a/src/test-network.c b/src/test-network.c -index 339743d..dde13f1 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -192,6 +192,16 @@ int main(int argc, char** argv) { +- err = loc_network_is_subnet_of(network2, network1); +- if (err != 1) { ++ err = loc_network_is_subnet(network2, network1); ++ if (err) { + fprintf(stderr, "Subnet check 2 failed: %d\n", err); exit(EXIT_FAILURE); } ++ // Make subnets ++ struct loc_network* subnet1 = NULL; ++ struct loc_network* subnet2 = NULL; ++ ++ err = loc_network_subnets(network1, &subnet1, &subnet2); ++ if (err || !subnet1 || !subnet2) { ++ fprintf(stderr, "Could not find subnets of network: %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ char* s = loc_network_str(subnet1); ++ printf("Received subnet1 = %s\n", s); ++ free(s); ++ ++ s = loc_network_str(subnet2); ++ printf("Received subnet2 = %s\n", s); ++ free(s); ++ ++ if (!loc_network_is_subnet(network1, subnet1)) { ++ fprintf(stderr, "Subnet1 is not a subnet\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (!loc_network_is_subnet(network1, subnet2)) { ++ fprintf(stderr, "Subnet2 is not a subnet\n"); ++ exit(EXIT_FAILURE); ++ } ++ + if (!loc_network_overlaps(network1, subnet1)) { + fprintf(stderr, "Network1 does not seem to contain subnet1\n"); + exit(EXIT_FAILURE); @@ -10399,340 +3687,84 @@ index 339743d..dde13f1 100644 + exit(EXIT_FAILURE); + } + - loc_network_unref(subnet1); - loc_network_unref(subnet2); - --- -2.20.1 - -From 0a39d5499ae0c343a6600ea30de7d95e384ef911 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 15:11:21 +0000 -Subject: [PATCH 103/111] networks: Remove comparing family - -Everything is encoded in IPv6 anyways... - -Signed-off-by: Michael Tremer ---- - src/network.c | 17 ----------------- - 1 file changed, 17 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 394bafc..66b460f 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -442,12 +442,6 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag - } - - LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) { -- // Compare family -- if (self->family > other->family) -- return 1; -- else if (self->family < other->family) -- return -1; -- - // Compare address - int r = in6_addr_cmp(&self->first_address, &other->first_address); - if (r) -@@ -482,10 +476,6 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network - } - - LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { -- // Check family -- if (self->family != other->family) -- return 0; -- - // The prefix must be smaller (this avoids the more complex comparisons later) - if (self->prefix > other->prefix) - return 0; -@@ -601,13 +591,6 @@ ERROR: - - static int __loc_network_exclude_to_list(struct loc_network* self, - struct loc_network* other, struct loc_network_list* list) { -- // Family must match -- if (self->family != other->family) { -- DEBUG(self->ctx, "Family mismatch\n"); -- -- return 1; -- } -- - // Other must be a subnet of self - if (!loc_network_is_subnet(self, other)) { - DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); --- -2.20.1 - -From abf559267696061a0c9723d8f8e09c9d1aa8fb75 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 15:13:08 +0000 -Subject: [PATCH 104/111] network: Do not execute with an error when the - excluded result will be empty - -Signed-off-by: Michael Tremer ---- - src/network.c | 6 ++++-- - 1 file changed, 4 insertions(+), 2 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 66b460f..9aa802f 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -595,14 +595,16 @@ static int __loc_network_exclude_to_list(struct loc_network* self, - if (!loc_network_is_subnet(self, other)) { - DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); - -- return 1; -+ // Exit silently -+ return 0; - } - - // We cannot perform this operation if both networks equal - if (loc_network_cmp(self, other) == 0) { - DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); - -- return 1; -+ // Exit silently -+ return 0; - } - - return __loc_network_exclude(self, other, list); --- -2.20.1 - -From 4fc034e20fdc36267f6ba18e27776fed747fe983 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 15:14:31 +0000 -Subject: [PATCH 105/111] network: Add more excluded networks straight to the - to_check list - -Signed-off-by: Michael Tremer ---- - src/network.c | 6 +----- - 1 file changed, 1 insertion(+), 5 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 9aa802f..19c387d 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -705,11 +705,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - if (loc_network_overlaps(subnet_to_check, subnet)) { - passed = 0; - -- struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet); -- if (excluded) { -- loc_network_list_merge(to_check, excluded); -- loc_network_list_unref(excluded); -- } -+ __loc_network_exclude_to_list(subnet_to_check, subnet, to_check); - - loc_network_unref(subnet); - break; --- -2.20.1 - -From 4de8ff8e5435225d3375a168054490d2a66b1baf Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 15:15:33 +0000 -Subject: [PATCH 106/111] network: Call subnet function with the correct order - of arguments - -Signed-off-by: Michael Tremer ---- - src/network.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 19c387d..a96ce6d 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -695,14 +695,14 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - subnet = loc_network_list_get(list, i); - - // Drop this subnet if is a subnet of another subnet -- if (loc_network_is_subnet(subnet_to_check, subnet)) { -+ if (loc_network_is_subnet(subnet, subnet_to_check)) { - passed = 0; - loc_network_unref(subnet); - break; - } - - // Break it down if it overlaps -- if (loc_network_overlaps(subnet_to_check, subnet)) { -+ if (loc_network_overlaps(subnet, subnet_to_check)) { - passed = 0; - - __loc_network_exclude_to_list(subnet_to_check, subnet, to_check); --- -2.20.1 - -From 058e7800e56150bcf0fb8dceaae6fbab1ed239e4 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 15:16:06 +0000 -Subject: [PATCH 107/111] network: Massively improve performance on exclude - -When we check the result for any overlaps, we can cut this short -by walking through both lists from start to end and remember the -last network that we checked. - -The next one will by definition be strictly greater and therefore -we do not need to check anything before this any more. - -Signed-off-by: Michael Tremer ---- - src/network.c | 19 +++++++++++++++++-- - 1 file changed, 17 insertions(+), 2 deletions(-) - -diff --git a/src/network.c b/src/network.c -index a96ce6d..10fa997 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -679,8 +679,10 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - return NULL; - } - -+ off_t smallest_subnet = 0; ++ loc_network_unref(subnet1); ++ loc_network_unref(subnet2); + - while (!loc_network_list_empty(to_check)) { -- struct loc_network* subnet_to_check = loc_network_list_pop(to_check); -+ struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check); - - // Check whether the subnet to check is part of the input list - if (loc_network_list_contains(list, subnet_to_check)) { -@@ -691,7 +693,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - // Marks whether this subnet passed all checks - int passed = 1; - -- for (unsigned int i = 0; i < loc_network_list_size(list); i++) { -+ for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) { - subnet = loc_network_list_get(list, i); - - // Drop this subnet if is a subnet of another subnet -@@ -711,6 +713,19 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - break; - } - -+ // If the subnet is strictly greater, we do not need to continue the search -+ r = loc_network_cmp(subnet, subnet_to_check); -+ if (r > 0) { -+ loc_network_unref(subnet); -+ break; ++ struct loc_network_list* excluded = loc_network_exclude(network1, network2); ++ if (!excluded) { ++ fprintf(stderr, "Could not create excluded list\n"); ++ exit(EXIT_FAILURE); ++ } + -+ // If it is strictly smaller, we can continue the search from here next -+ // time because all networks that are to be checked can only be larger -+ // than this one. -+ } else if (r < 0) { -+ smallest_subnet = i; -+ } ++ loc_network_list_dump(excluded); ++ loc_network_list_unref(excluded); + - loc_network_unref(subnet); - } - --- -2.20.1 - -From 9caf6cf5f0fd5eb56ad1678910dbc8017f48863c Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 15:17:21 +0000 -Subject: [PATCH 108/111] network: Remove deprecated sort call - -Signed-off-by: Michael Tremer ---- - src/network.c | 3 --- - 1 file changed, 3 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 10fa997..4d7ac79 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -738,9 +738,6 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - - loc_network_list_unref(to_check); + // Create a database + struct loc_writer* writer; + err = loc_writer_new(ctx, &writer, NULL, NULL); +@@ -160,6 +243,28 @@ int main(int argc, char** argv) { + // Set ASN + loc_network_set_asn(network4, 1024); -- // Sort the result -- loc_network_list_sort(subnets); -- - return subnets; - } ++ // Try adding an invalid network ++ struct loc_network* network; ++ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32"); ++ if (err != -EINVAL) { ++ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Try adding a single address ++ err = loc_writer_add_network(writer, &network, "2001:db8::"); ++ if (err) { ++ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Try adding localhost ++ err = loc_writer_add_network(writer, &network, "::1/128"); ++ if (err != -EINVAL) { ++ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +@@ -177,7 +282,10 @@ int main(int argc, char** argv) { + loc_network_unref(network2); + loc_network_unref(network3); + loc_network_unref(network4); ++ ++#if 0 + loc_network_tree_unref(tree); ++#endif --- -2.20.1 - -From 8ebf1d3912020d5c0ed98bb32a0640952bd6447c Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 15:17:42 +0000 -Subject: [PATCH 109/111] network-list: Include network header - -Signed-off-by: Michael Tremer ---- - src/loc/network-list.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/src/loc/network-list.h b/src/loc/network-list.h -index 21c7402..bee21c4 100644 ---- a/src/loc/network-list.h -+++ b/src/loc/network-list.h -@@ -17,6 +17,8 @@ - #ifndef LIBLOC_NETWORK_LIST_H - #define LIBLOC_NETWORK_LIST_H + // And open it again from disk + struct loc_database* db; +diff --git a/src/writer.c b/src/writer.c +index 5939cff..2f09b56 100644 +--- a/src/writer.c ++++ b/src/writer.c +@@ -147,8 +147,19 @@ static void loc_writer_free(struct loc_writer* writer) { + EVP_PKEY_free(writer->private_key2); -+#include + // Unref all AS +- for (unsigned int i = 0; i < writer->as_count; i++) { +- loc_as_unref(writer->as[i]); ++ if (writer->as) { ++ for (unsigned int i = 0; i < writer->as_count; i++) { ++ loc_as_unref(writer->as[i]); ++ } ++ free(writer->as); ++ } + - struct loc_network_list; - int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); - struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); --- -2.20.1 - -From fff093b6827a31ff84a6d5150bfede140b696d25 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 15:24:43 +0000 -Subject: [PATCH 110/111] network-list: Use clear function to tidy up - -Signed-off-by: Michael Tremer ---- - src/network-list.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/network-list.c b/src/network-list.c -index bd3d64e..f3458b4 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -70,8 +70,8 @@ LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list - static void loc_network_list_free(struct loc_network_list* list) { - DEBUG(list->ctx, "Releasing network list at %p\n", list); - -- for (unsigned int i = 0; i < list->size; i++) -- loc_network_unref(list->elements[i]); -+ // Remove all content -+ loc_network_list_clear(list); ++ // Unref all countries ++ if (writer->countries) { ++ for (unsigned int i = 0; i < writer->countries_count; i++) { ++ loc_country_unref(writer->countries[i]); ++ } ++ free(writer->countries); + } - loc_unref(list->ctx); - free(list); --- -2.20.1 - -From adbec5c0317a1c007a897fe3f63f0f86ff448124 Mon Sep 17 00:00:00 2001 -From: Michael Tremer -Date: Wed, 25 Nov 2020 20:00:46 +0000 -Subject: [PATCH 111/111] configure: Bump version to 0.9.5 - -Signed-off-by: Michael Tremer ---- - configure.ac | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 2364dfd..012d8ca 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -1,6 +1,6 @@ - AC_PREREQ(2.60) - AC_INIT([libloc], -- [0.9.4], -+ [0.9.5], - [location@lists.ipfire.org], - [libloc], - [https://location.ipfire.org/]) --- -2.20.1 - + // Release network tree -- 2.39.5