-From ee6ea3986dc80183157f67275dc9f28231b5d5b2 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 24 Sep 2020 10:17:58 +0000
-Subject: [PATCH 001/111] Revert "importer: Purge any redundant entries"
-
-This reverts commit c2cc55d5a6875c3838f060032eaed89dcfb92ef6.
-
-The query stalls the database and therefore the automatic
-scripts are no longer able to generate a new version of the
-database.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 22 +---------------------
- 1 file changed, 1 insertion(+), 21 deletions(-)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index 1467923..e3a07a0 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -374,27 +374,7 @@ class CLI(object):
- INSERT INTO autnums(number, name)
- SELECT _autnums.number, _organizations.name FROM _autnums
- JOIN _organizations ON _autnums.organization = _organizations.handle
-- ON CONFLICT (number) DO UPDATE SET name = excluded.name
-- """)
--
-- self.db.execute("""
-- --- Purge any redundant entries
-- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
-- AS
-- SELECT network FROM networks candidates
-- WHERE EXISTS (
-- SELECT FROM networks
-- WHERE
-- networks.network << candidates.network
-- AND
-- networks.country = candidates.country
-- );
--
-- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
--
-- DELETE FROM networks WHERE EXISTS (
-- SELECT FROM _garbage WHERE networks.network = _garbage.network
-- );
-+ ON CONFLICT (number) DO UPDATE SET name = excluded.name;
- """)
-
- # Download all extended sources
---
-2.20.1
-
-From 92f6abf4e272672bb0a71cfe991261b95ebe2fef Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 24 Sep 2020 10:18:58 +0000
-Subject: [PATCH 002/111] Revert "importer: Import raw sources for inetnum's
- again"
-
-This reverts commit 64e95fa903edec8b4e4e59830b395e2e4a411853.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/importer.py | 14 ++++----
- src/python/location-importer.in | 63 ---------------------------------
- 2 files changed, 7 insertions(+), 70 deletions(-)
-
-diff --git a/src/python/importer.py b/src/python/importer.py
-index f19db4b..de20f37 100644
---- a/src/python/importer.py
-+++ b/src/python/importer.py
-@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
- "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
-
- # Asia Pacific Network Information Centre
-- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
-+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
- #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
- #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
- "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
-@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
- # XXX ???
-
- # Réseaux IP Européens
-- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
-+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
-@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
-
- EXTENDED_SOURCES = (
- # African Network Information Centre
-- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
-+ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
-
- # Asia Pacific Network Information Centre
-- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
-+ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
-
- # American Registry for Internet Numbers
- "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
-@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
-
- # Réseaux IP Européens
-- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
-+ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
- )
-
- class Downloader(object):
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index e3a07a0..77952f2 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -393,10 +393,6 @@ class CLI(object):
- if line.startswith("aut-num:"):
- return self._parse_autnum_block(block)
-
-- # inetnum
-- if line.startswith("inet6num:") or line.startswith("inetnum:"):
-- return self._parse_inetnum_block(block)
--
- # organisation
- elif line.startswith("organisation:"):
- return self._parse_org_block(block)
-@@ -426,65 +422,6 @@ class CLI(object):
- autnum.get("asn"), autnum.get("org"),
- )
-
-- def _parse_inetnum_block(self, block):
-- logging.debug("Parsing inetnum block:")
--
-- inetnum = {}
-- for line in block:
-- logging.debug(line)
--
-- # Split line
-- key, val = split_line(line)
--
-- if key == "inetnum":
-- start_address, delim, end_address = val.partition("-")
--
-- # Strip any excess space
-- start_address, end_address = start_address.rstrip(), end_address.strip()
--
-- # Convert to IP address
-- try:
-- start_address = ipaddress.ip_address(start_address)
-- end_address = ipaddress.ip_address(end_address)
-- except ValueError:
-- logging.warning("Could not parse line: %s" % line)
-- return
--
-- # Set prefix to default
-- prefix = 32
--
-- # Count number of addresses in this subnet
-- num_addresses = int(end_address) - int(start_address)
-- if num_addresses:
-- prefix -= math.log(num_addresses, 2)
--
-- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
--
-- elif key == "inet6num":
-- inetnum[key] = val
--
-- elif key == "country":
-- if val == "UNITED STATES":
-- val = "US"
--
-- inetnum[key] = val.upper()
--
-- # Skip empty objects
-- if not inetnum:
-- return
--
-- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
--
-- # Bail out in case we have processed a non-public IP network
-- if network.is_private:
-- logging.warning("Skipping non-globally routable network: %s" % network)
-- return
--
-- self.db.execute("INSERT INTO networks(network, country) \
-- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
-- "%s" % network, inetnum.get("country"),
-- )
--
- def _parse_org_block(self, block):
- org = {}
- for line in block:
---
-2.20.1
-
-From f532841e9197ce2f40aad8c086d786b2cb783a54 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Mon, 12 Oct 2020 20:53:31 +0000
-Subject: [PATCH 003/111] Revert "Revert "importer: Import raw sources for
- inetnum's again""
-
-This reverts commit 92f6abf4e272672bb0a71cfe991261b95ebe2fef.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/importer.py | 14 ++++----
- src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
- 2 files changed, 70 insertions(+), 7 deletions(-)
-
-diff --git a/src/python/importer.py b/src/python/importer.py
-index de20f37..f19db4b 100644
---- a/src/python/importer.py
-+++ b/src/python/importer.py
-@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
- "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
-
- # Asia Pacific Network Information Centre
-- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
-+ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-+ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
- #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
- #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
- "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
-@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
- # XXX ???
-
- # Réseaux IP Européens
-- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
-+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
-@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
-
- EXTENDED_SOURCES = (
- # African Network Information Centre
-- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
-+ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
-
- # Asia Pacific Network Information Centre
-- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
-+ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
-
- # American Registry for Internet Numbers
- "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
-@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
-
- # Réseaux IP Européens
-- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
-+ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
- )
+diff --git a/Makefile.am b/Makefile.am
+index a0431a6..ebd7e17 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -91,11 +91,14 @@ EXTRA_DIST += \
+ pkginclude_HEADERS = \
+ src/loc/libloc.h \
+ src/loc/as.h \
++ src/loc/as-list.h \
+ src/loc/compat.h \
+ src/loc/country.h \
++ src/loc/country-list.h \
+ src/loc/database.h \
+ src/loc/format.h \
+ src/loc/network.h \
++ src/loc/network-list.h \
+ src/loc/private.h \
+ src/loc/stringpool.h \
+ src/loc/resolv.h \
+@@ -107,9 +110,12 @@ lib_LTLIBRARIES = \
+ src_libloc_la_SOURCES = \
+ src/libloc.c \
+ src/as.c \
++ src/as-list.c \
+ src/country.c \
++ src/country-list.c \
+ src/database.c \
+ src/network.c \
++ src/network-list.c \
+ src/resolv.c \
+ src/stringpool.c \
+ src/writer.c
+@@ -312,6 +318,7 @@ check_PROGRAMS = \
+ src/test-database \
+ src/test-as \
+ src/test-network \
++ src/test-network-list \
+ src/test-country \
+ src/test-signature
- class Downloader(object):
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index 77952f2..e3a07a0 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -393,6 +393,10 @@ class CLI(object):
- if line.startswith("aut-num:"):
- return self._parse_autnum_block(block)
-
-+ # inetnum
-+ if line.startswith("inet6num:") or line.startswith("inetnum:"):
-+ return self._parse_inetnum_block(block)
-+
- # organisation
- elif line.startswith("organisation:"):
- return self._parse_org_block(block)
-@@ -422,6 +426,65 @@ class CLI(object):
- autnum.get("asn"), autnum.get("org"),
- )
+@@ -351,6 +358,15 @@ src_test_network_CFLAGS = \
+ src_test_network_LDADD = \
+ src/libloc.la
-+ def _parse_inetnum_block(self, block):
-+ logging.debug("Parsing inetnum block:")
++src_test_network_list_SOURCES = \
++ src/test-network-list.c
+
-+ inetnum = {}
-+ for line in block:
-+ logging.debug(line)
++src_test_network_list_CFLAGS = \
++ $(TESTS_CFLAGS)
+
-+ # Split line
-+ key, val = split_line(line)
++src_test_network_list_LDADD = \
++ src/libloc.la
+
-+ if key == "inetnum":
-+ start_address, delim, end_address = val.partition("-")
+ src_test_stringpool_SOURCES = \
+ src/test-stringpool.c
+
+diff --git a/configure.ac b/configure.ac
+index 2364dfd..012d8ca 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -1,6 +1,6 @@
+ AC_PREREQ(2.60)
+ AC_INIT([libloc],
+- [0.9.4],
++ [0.9.5],
+ [location@lists.ipfire.org],
+ [libloc],
+ [https://location.ipfire.org/])
+diff --git a/src/.gitignore b/src/.gitignore
+index caf80b5..3ccbdb8 100644
+--- a/src/.gitignore
++++ b/src/.gitignore
+@@ -10,5 +10,6 @@ test-libloc
+ test-database
+ test-country
+ test-network
++test-network-list
+ test-signature
+ test-stringpool
+diff --git a/src/as-list.c b/src/as-list.c
+new file mode 100644
+index 0000000..5acbb8a
+--- /dev/null
++++ b/src/as-list.c
+@@ -0,0 +1,161 @@
++/*
++ libloc - A library to determine the location of someone on the Internet
+
-+ # Strip any excess space
-+ start_address, end_address = start_address.rstrip(), end_address.strip()
++ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
+
-+ # Convert to IP address
-+ try:
-+ start_address = ipaddress.ip_address(start_address)
-+ end_address = ipaddress.ip_address(end_address)
-+ except ValueError:
-+ logging.warning("Could not parse line: %s" % line)
-+ return
++ This library is free software; you can redistribute it and/or
++ modify it under the terms of the GNU Lesser General Public
++ License as published by the Free Software Foundation; either
++ version 2.1 of the License, or (at your option) any later version.
+
-+ # Set prefix to default
-+ prefix = 32
++ This library is distributed in the hope that it will be useful,
++ but WITHOUT ANY WARRANTY; without even the implied warranty of
++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ Lesser General Public License for more details.
++*/
+
-+ # Count number of addresses in this subnet
-+ num_addresses = int(end_address) - int(start_address)
-+ if num_addresses:
-+ prefix -= math.log(num_addresses, 2)
++#include <errno.h>
++#include <stdlib.h>
+
-+ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
++#include <loc/as.h>
++#include <loc/as-list.h>
++#include <loc/private.h>
+
-+ elif key == "inet6num":
-+ inetnum[key] = val
++struct loc_as_list {
++ struct loc_ctx* ctx;
++ int refcount;
+
-+ elif key == "country":
-+ if val == "UNITED STATES":
-+ val = "US"
++ struct loc_as** elements;
++ size_t elements_size;
+
-+ inetnum[key] = val.upper()
++ size_t size;
++};
+
-+ # Skip empty objects
-+ if not inetnum:
-+ return
++static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
++ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
++ list, size, list->elements_size + size);
+
-+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
++ struct loc_as** elements = reallocarray(list->elements,
++ list->elements_size + size, sizeof(*list->elements));
++ if (!elements)
++ return -errno;
+
-+ # Bail out in case we have processed a non-public IP network
-+ if network.is_private:
-+ logging.warning("Skipping non-globally routable network: %s" % network)
-+ return
++ list->elements = elements;
++ list->elements_size += size;
+
-+ self.db.execute("INSERT INTO networks(network, country) \
-+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
-+ "%s" % network, inetnum.get("country"),
-+ )
++ return 0;
++}
+
- def _parse_org_block(self, block):
- org = {}
- for line in block:
---
-2.20.1
-
-From a36bc686865fc87ea386fd90b389338bdcb80cbc Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Mon, 12 Oct 2020 20:53:32 +0000
-Subject: [PATCH 004/111] location-importer.in: only import relevant data from
- AFRINIC, APNIC and RIPE
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-In contrast to ARIN and LACNIC, we are able to process more detailled
-feeds from those RIRs, avoiding storage of obviously unnecessary data.
-
-Thanks to various SQL optimisations, doing so now takes less time than
-the first version of this did.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
- 1 file changed, 87 insertions(+), 2 deletions(-)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index e3a07a0..093f325 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -165,6 +165,7 @@ class CLI(object):
- -- networks
- CREATE TABLE IF NOT EXISTS networks(network inet, country text);
- CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
-+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
- CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
-
- -- overrides
-@@ -363,6 +364,16 @@ class CLI(object):
- CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
- ON COMMIT DROP;
- CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
++LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
++ struct loc_as_list** list) {
++ struct loc_as_list* l = calloc(1, sizeof(*l));
++ if (!l)
++ return -ENOMEM;
+
-+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
-+ ON COMMIT DROP;
-+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
-+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
-+ """)
++ l->ctx = loc_ref(ctx);
++ l->refcount = 1;
+
-+ # Remove all previously imported content
-+ self.db.execute("""
-+ TRUNCATE TABLE networks;
- """)
-
- for source in location.importer.WHOIS_SOURCES:
-@@ -370,6 +381,67 @@ class CLI(object):
- for block in f:
- self._parse_block(block)
-
-+ # Process all parsed networks from every RIR we happen to have access to,
-+ # insert the largest network chunks into the networks table immediately...
-+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
++ DEBUG(l->ctx, "AS list allocated at %p\n", l);
++ *list = l;
+
-+ for family in (row.family for row in families):
-+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
++ return 0;
++}
+
-+ self.db.execute("INSERT INTO networks(network, country) \
-+ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
++LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
++ list->refcount++;
+
-+ # ... determine any other prefixes for this network family, ...
-+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
-+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
++ return list;
++}
+
-+ # ... and insert networks with this prefix in case they provide additional
-+ # information (i. e. subnet of a larger chunk with a different country)
-+ for prefix in (row.prefix for row in prefixes):
-+ self.db.execute("""
-+ WITH candidates AS (
-+ SELECT
-+ _rirdata.network,
-+ _rirdata.country
-+ FROM
-+ _rirdata
-+ WHERE
-+ family(_rirdata.network) = %s
-+ AND
-+ masklen(_rirdata.network) = %s
-+ ),
-+ filtered AS (
-+ SELECT
-+ DISTINCT ON (c.network)
-+ c.network,
-+ c.country,
-+ masklen(networks.network),
-+ networks.country AS parent_country
-+ FROM
-+ candidates c
-+ LEFT JOIN
-+ networks
-+ ON
-+ c.network << networks.network
-+ ORDER BY
-+ c.network,
-+ masklen(networks.network) DESC NULLS LAST
-+ )
-+ INSERT INTO
-+ networks(network, country)
-+ SELECT
-+ network,
-+ country
-+ FROM
-+ filtered
-+ WHERE
-+ parent_country IS NULL
-+ OR
-+ country <> parent_country
-+ ON CONFLICT DO NOTHING""",
-+ family, prefix,
-+ )
++static void loc_as_list_free(struct loc_as_list* list) {
++ DEBUG(list->ctx, "Releasing AS list at %p\n", list);
+
- self.db.execute("""
- INSERT INTO autnums(number, name)
- SELECT _autnums.number, _organizations.name FROM _autnums
-@@ -470,17 +542,30 @@ class CLI(object):
- inetnum[key] = val.upper()
-
- # Skip empty objects
-- if not inetnum:
-+ if not inetnum or not "country" in inetnum:
- return
-
- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-
-+ # Bail out in case we have processed a network covering the entire IP range, which
-+ # is necessary to work around faulty (?) IPv6 network processing
-+ if network.prefixlen == 0:
-+ logging.warning("Skipping network covering the entire IP adress range: %s" % network)
-+ return
++ loc_as_list_clear(list);
+
-+ # Bail out in case we have processed a network whose prefix length indicates it is
-+ # not globally routable (we have decided not to process them at the moment, as they
-+ # significantly enlarge our database without providing very helpful additional information)
-+ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
-+ logging.info("Skipping network too small to be publicly announced: %s" % network)
-+ return
++ loc_unref(list->ctx);
++ free(list);
++}
+
- # Bail out in case we have processed a non-public IP network
- if network.is_private:
- logging.warning("Skipping non-globally routable network: %s" % network)
- return
-
-- self.db.execute("INSERT INTO networks(network, country) \
-+ self.db.execute("INSERT INTO _rirdata(network, country) \
- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
- "%s" % network, inetnum.get("country"),
- )
---
-2.20.1
-
-From 2373de384f10f5573bbd7570f5522545df70c0e3 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 16 Oct 2020 12:24:58 +0000
-Subject: [PATCH 005/111] location-importer: Include all overridden networks
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 2 ++
- 1 file changed, 2 insertions(+)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index 093f325..d249a35 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -240,6 +240,8 @@ class CLI(object):
- SELECT network FROM announcements
- UNION
- SELECT network FROM networks
-+ UNION
-+ SELECT network FROM network_overrides
- ORDER BY network
- )
-
---
-2.20.1
-
-From 13f67f285856e8eabfeff2daf1be3aeaa36a82cc Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 16 Oct 2020 12:26:38 +0000
-Subject: [PATCH 006/111] Revert "location-importer.in: only import relevant
- data from AFRINIC, APNIC and RIPE"
-
-This reverts commit a36bc686865fc87ea386fd90b389338bdcb80cbc.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 89 +--------------------------------
- 1 file changed, 2 insertions(+), 87 deletions(-)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index d249a35..b220eaf 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -165,7 +165,6 @@ class CLI(object):
- -- networks
- CREATE TABLE IF NOT EXISTS networks(network inet, country text);
- CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
-- CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
- CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
-
- -- overrides
-@@ -366,16 +365,6 @@ class CLI(object):
- CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
- ON COMMIT DROP;
- CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
--
-- CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
-- ON COMMIT DROP;
-- CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
-- CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
-- """)
--
-- # Remove all previously imported content
-- self.db.execute("""
-- TRUNCATE TABLE networks;
- """)
-
- for source in location.importer.WHOIS_SOURCES:
-@@ -383,67 +372,6 @@ class CLI(object):
- for block in f:
- self._parse_block(block)
-
-- # Process all parsed networks from every RIR we happen to have access to,
-- # insert the largest network chunks into the networks table immediately...
-- families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
--
-- for family in (row.family for row in families):
-- smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
--
-- self.db.execute("INSERT INTO networks(network, country) \
-- SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
--
-- # ... determine any other prefixes for this network family, ...
-- prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
-- WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
--
-- # ... and insert networks with this prefix in case they provide additional
-- # information (i. e. subnet of a larger chunk with a different country)
-- for prefix in (row.prefix for row in prefixes):
-- self.db.execute("""
-- WITH candidates AS (
-- SELECT
-- _rirdata.network,
-- _rirdata.country
-- FROM
-- _rirdata
-- WHERE
-- family(_rirdata.network) = %s
-- AND
-- masklen(_rirdata.network) = %s
-- ),
-- filtered AS (
-- SELECT
-- DISTINCT ON (c.network)
-- c.network,
-- c.country,
-- masklen(networks.network),
-- networks.country AS parent_country
-- FROM
-- candidates c
-- LEFT JOIN
-- networks
-- ON
-- c.network << networks.network
-- ORDER BY
-- c.network,
-- masklen(networks.network) DESC NULLS LAST
-- )
-- INSERT INTO
-- networks(network, country)
-- SELECT
-- network,
-- country
-- FROM
-- filtered
-- WHERE
-- parent_country IS NULL
-- OR
-- country <> parent_country
-- ON CONFLICT DO NOTHING""",
-- family, prefix,
-- )
--
- self.db.execute("""
- INSERT INTO autnums(number, name)
- SELECT _autnums.number, _organizations.name FROM _autnums
-@@ -544,30 +472,17 @@ class CLI(object):
- inetnum[key] = val.upper()
-
- # Skip empty objects
-- if not inetnum or not "country" in inetnum:
-+ if not inetnum:
- return
-
- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-
-- # Bail out in case we have processed a network covering the entire IP range, which
-- # is necessary to work around faulty (?) IPv6 network processing
-- if network.prefixlen == 0:
-- logging.warning("Skipping network covering the entire IP adress range: %s" % network)
-- return
--
-- # Bail out in case we have processed a network whose prefix length indicates it is
-- # not globally routable (we have decided not to process them at the moment, as they
-- # significantly enlarge our database without providing very helpful additional information)
-- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
-- logging.info("Skipping network too small to be publicly announced: %s" % network)
-- return
--
- # Bail out in case we have processed a non-public IP network
- if network.is_private:
- logging.warning("Skipping non-globally routable network: %s" % network)
- return
-
-- self.db.execute("INSERT INTO _rirdata(network, country) \
-+ self.db.execute("INSERT INTO networks(network, country) \
- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
- "%s" % network, inetnum.get("country"),
- )
---
-2.20.1
-
-From 44341478233115b26bb27fdb24da5b0a1eedb173 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 16 Oct 2020 12:26:43 +0000
-Subject: [PATCH 007/111] Revert "Revert "Revert "importer: Import raw sources
- for inetnum's again"""
-
-This reverts commit f532841e9197ce2f40aad8c086d786b2cb783a54.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/importer.py | 14 ++++----
- src/python/location-importer.in | 63 ---------------------------------
- 2 files changed, 7 insertions(+), 70 deletions(-)
-
-diff --git a/src/python/importer.py b/src/python/importer.py
-index f19db4b..de20f37 100644
---- a/src/python/importer.py
-+++ b/src/python/importer.py
-@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
- "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
-
- # Asia Pacific Network Information Centre
-- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
-+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
- #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
- #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
- "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
-@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
- # XXX ???
-
- # Réseaux IP Européens
-- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
-+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
-@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
-
- EXTENDED_SOURCES = (
- # African Network Information Centre
-- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
-+ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
-
- # Asia Pacific Network Information Centre
-- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
-+ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
-
- # American Registry for Internet Numbers
- "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
-@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
-
- # Réseaux IP Européens
-- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
-+ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
- )
-
- class Downloader(object):
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index b220eaf..e87d378 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -395,10 +395,6 @@ class CLI(object):
- if line.startswith("aut-num:"):
- return self._parse_autnum_block(block)
-
-- # inetnum
-- if line.startswith("inet6num:") or line.startswith("inetnum:"):
-- return self._parse_inetnum_block(block)
--
- # organisation
- elif line.startswith("organisation:"):
- return self._parse_org_block(block)
-@@ -428,65 +424,6 @@ class CLI(object):
- autnum.get("asn"), autnum.get("org"),
- )
-
-- def _parse_inetnum_block(self, block):
-- logging.debug("Parsing inetnum block:")
--
-- inetnum = {}
-- for line in block:
-- logging.debug(line)
--
-- # Split line
-- key, val = split_line(line)
--
-- if key == "inetnum":
-- start_address, delim, end_address = val.partition("-")
--
-- # Strip any excess space
-- start_address, end_address = start_address.rstrip(), end_address.strip()
--
-- # Convert to IP address
-- try:
-- start_address = ipaddress.ip_address(start_address)
-- end_address = ipaddress.ip_address(end_address)
-- except ValueError:
-- logging.warning("Could not parse line: %s" % line)
-- return
--
-- # Set prefix to default
-- prefix = 32
--
-- # Count number of addresses in this subnet
-- num_addresses = int(end_address) - int(start_address)
-- if num_addresses:
-- prefix -= math.log(num_addresses, 2)
--
-- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
--
-- elif key == "inet6num":
-- inetnum[key] = val
--
-- elif key == "country":
-- if val == "UNITED STATES":
-- val = "US"
--
-- inetnum[key] = val.upper()
--
-- # Skip empty objects
-- if not inetnum:
-- return
--
-- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
--
-- # Bail out in case we have processed a non-public IP network
-- if network.is_private:
-- logging.warning("Skipping non-globally routable network: %s" % network)
-- return
--
-- self.db.execute("INSERT INTO networks(network, country) \
-- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
-- "%s" % network, inetnum.get("country"),
-- )
--
- def _parse_org_block(self, block):
- org = {}
- for line in block:
---
-2.20.1
-
-From a7d3a7a0565a0e09d3442e5829a0f30f016993b9 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 20 Oct 2020 20:44:43 +0000
-Subject: [PATCH 008/111] as: Fix dereferencing NULL pointer when setting AS
- name
-
-Reported-by: Gisle Vanem <gisle.vanem@gmail.com>
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/as.c | 8 +++++++-
- 1 file changed, 7 insertions(+), 1 deletion(-)
-
++LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
++ if (!list)
++ return NULL;
++
++ if (--list->refcount > 0)
++ return list;
++
++ loc_as_list_free(list);
++ return NULL;
++}
++
++LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
++ return list->size;
++}
++
++LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
++ return list->size == 0;
++}
++
++LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
++ if (!list->elements)
++ return;
++
++ for (unsigned int i = 0; i < list->size; i++)
++ loc_as_unref(list->elements[i]);
++
++ free(list->elements);
++ list->elements = NULL;
++ list->elements_size = 0;
++
++ list->size = 0;
++}
++
++LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
++ // Check index
++ if (index >= list->size)
++ return NULL;
++
++ return loc_as_ref(list->elements[index]);
++}
++
++LOC_EXPORT int loc_as_list_append(
++ struct loc_as_list* list, struct loc_as* as) {
++ if (loc_as_list_contains(list, as))
++ return 0;
++
++ // Check if we have space left
++ if (list->size >= list->elements_size) {
++ int r = loc_as_list_grow(list, 64);
++ if (r)
++ return r;
++ }
++
++ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
++
++ list->elements[list->size++] = loc_as_ref(as);
++
++ return 0;
++}
++
++LOC_EXPORT int loc_as_list_contains(
++ struct loc_as_list* list, struct loc_as* as) {
++ for (unsigned int i = 0; i < list->size; i++) {
++ if (loc_as_cmp(as, list->elements[i]) == 0)
++ return 1;
++ }
++
++ return 0;
++}
++
++LOC_EXPORT int loc_as_list_contains_number(
++ struct loc_as_list* list, uint32_t number) {
++ struct loc_as* as;
++
++ int r = loc_as_new(list->ctx, &as, number);
++ if (r)
++ return -1;
++
++ r = loc_as_list_contains(list, as);
++ loc_as_unref(as);
++
++ return r;
++}
diff --git a/src/as.c b/src/as.c
-index e1fbb01..8421ac8 100644
+index e1fbb01..757bf3d 100644
--- a/src/as.c
+++ b/src/as.c
@@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
return 0;
}
---
-2.20.1
-
-From ddb326ad38a7c7202315dd2c6f938313db04ee22 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 21 Oct 2020 09:18:08 +0000
-Subject: [PATCH 009/111] as: Do not attempt to match name when it wasn't set
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/as.c | 4 ++++
- 1 file changed, 4 insertions(+)
-
-diff --git a/src/as.c b/src/as.c
-index 8421ac8..757bf3d 100644
---- a/src/as.c
-+++ b/src/as.c
-@@ -145,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
+@@ -139,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
if (!string)
return 1;
// Search if string is in name
if (strcasestr(as->name, string) != NULL)
return 1;
---
-2.20.1
-
-From d226ad2d97cbcd42ce807d9308569b1b9c5d4e2f Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 21 Oct 2020 09:28:39 +0000
-Subject: [PATCH 010/111] writer: Free array with pointer to ASes, too
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/writer.c | 7 +++++--
- 1 file changed, 5 insertions(+), 2 deletions(-)
-
-diff --git a/src/writer.c b/src/writer.c
-index 5939cff..160650f 100644
---- a/src/writer.c
-+++ b/src/writer.c
-@@ -147,8 +147,11 @@ static void loc_writer_free(struct loc_writer* writer) {
- EVP_PKEY_free(writer->private_key2);
-
- // Unref all AS
-- for (unsigned int i = 0; i < writer->as_count; i++) {
-- loc_as_unref(writer->as[i]);
-+ if (writer->as) {
-+ for (unsigned int i = 0; i < writer->as_count; i++) {
-+ loc_as_unref(writer->as[i]);
-+ }
-+ free(writer->as);
- }
-
- // Release network tree
---
-2.20.1
-
-From d89a7d62772048ae1bd18d03f69df46b7e1a3d3c Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 21 Oct 2020 09:31:29 +0000
-Subject: [PATCH 011/111] writer: Free countries when the writer is being
- destroyed
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/writer.c | 8 ++++++++
- 1 file changed, 8 insertions(+)
-
-diff --git a/src/writer.c b/src/writer.c
-index 160650f..2f09b56 100644
---- a/src/writer.c
-+++ b/src/writer.c
-@@ -154,6 +154,14 @@ static void loc_writer_free(struct loc_writer* writer) {
- free(writer->as);
- }
-
-+ // Unref all countries
-+ if (writer->countries) {
-+ for (unsigned int i = 0; i < writer->countries_count; i++) {
-+ loc_country_unref(writer->countries[i]);
-+ }
-+ free(writer->countries);
-+ }
-+
- // Release network tree
- if (writer->networks)
- loc_network_tree_unref(writer->networks);
---
-2.20.1
-
-From 0f1aedbc68e3945770c93e0ebd83eed0f555d6f0 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 21 Oct 2020 13:19:44 +0000
-Subject: [PATCH 012/111] tests: Try adding an invalid network
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/test-network.c | 8 ++++++++
- 1 file changed, 8 insertions(+)
-
-diff --git a/src/test-network.c b/src/test-network.c
-index d38f13d..e908b57 100644
---- a/src/test-network.c
-+++ b/src/test-network.c
-@@ -160,6 +160,14 @@ int main(int argc, char** argv) {
- // Set ASN
- loc_network_set_asn(network4, 1024);
-
-+ // Try adding an invalid network
-+ struct loc_network* network;
-+ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
-+ if (err != -EINVAL) {
-+ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
-+ exit(EXIT_FAILURE);
-+ }
-+
- FILE* f = tmpfile();
- if (!f) {
- fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
---
-2.20.1
-
-From 13ad6e695f9ffc7847b3afe3e9cbcea8fb3a443f Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 21 Oct 2020 13:36:35 +0000
-Subject: [PATCH 013/111] networks: Improve parsing IP addresses
-
-loc_network_new_from_string() seem to have had some unexpected
-behaviour for invalid inputs.
-
-The function has been tidied up slightly and returns as soon as
-some invalid input was detected.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 52 +++++++++++++++++++++++++++++++---------------
- src/test-network.c | 7 +++++++
- 2 files changed, 42 insertions(+), 17 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index 366caa2..c112a41 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -160,9 +160,10 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
- LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
- const char* address_string) {
- struct in6_addr first_address;
-- unsigned int prefix = 0;
- char* prefix_string;
-- int r = 1;
-+ int r = -EINVAL;
-+
-+ DEBUG(ctx, "Attempting to parse network %s\n", address_string);
-
- // Make a copy of the string to work on it
- char* buffer = strdup(address_string);
-@@ -171,29 +172,46 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
- // Split address and prefix
- address_string = strsep(&prefix_string, "/");
-
-- // Did we find a prefix?
-- if (prefix_string) {
-- // Convert prefix to integer
-- prefix = strtol(prefix_string, NULL, 10);
-+ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
-
-- if (prefix) {
-- // Parse the address
-- r = loc_parse_address(ctx, address_string, &first_address);
-+ // We need to have a prefix
-+ if (!prefix_string) {
-+ DEBUG(ctx, "No prefix set\n");
-+ goto FAIL;
-+ }
-
-- // Map the prefix to IPv6 if needed
-- if (IN6_IS_ADDR_V4MAPPED(&first_address))
-- prefix += 96;
-- }
-+ // Convert prefix to integer
-+ unsigned int prefix = strtol(prefix_string, NULL, 10);
-+
-+ // End if the prefix was invalid
-+ if (!prefix) {
-+ DEBUG(ctx, "The prefix is zero or not a number\n");
-+ goto FAIL;
-+ }
-+
-+ // Parse the address
-+ r = loc_parse_address(ctx, address_string, &first_address);
-+ if (r) {
-+ DEBUG(ctx, "The address could not be parsed\n");
-+ goto FAIL;
- }
-
-+ // Map the prefix to IPv6 if needed
-+ if (IN6_IS_ADDR_V4MAPPED(&first_address))
-+ prefix += 96;
-+
-+FAIL:
- // Free temporary buffer
- free(buffer);
-
-- if (r == 0) {
-- r = loc_network_new(ctx, network, &first_address, prefix);
-- }
-+ // Exit if the parsing was unsuccessful
-+ if (r)
-+ return r;
-+
-+ DEBUG(ctx, "GOT HERE\n");
-
-- return r;
-+ // Create a new network
-+ return loc_network_new(ctx, network, &first_address, prefix);
- }
-
- LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
-diff --git a/src/test-network.c b/src/test-network.c
-index e908b57..85eca00 100644
---- a/src/test-network.c
-+++ b/src/test-network.c
-@@ -168,6 +168,13 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
-+ // Try adding a single address
-+ err = loc_writer_add_network(writer, &network, "2001:db8::");
-+ if (err != -EINVAL) {
-+ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
-+ exit(EXIT_FAILURE);
-+ }
-+
- FILE* f = tmpfile();
- if (!f) {
- fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
---
-2.20.1
-
-From 6a467e9345bb5a3d37911c9aaac30019eaa4492b Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 21 Oct 2020 13:43:21 +0000
-Subject: [PATCH 014/111] networks: Test if we can add localhost (IPv6)
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/test-network.c | 7 +++++++
- 1 file changed, 7 insertions(+)
-
-diff --git a/src/test-network.c b/src/test-network.c
-index 85eca00..8c7e898 100644
---- a/src/test-network.c
-+++ b/src/test-network.c
-@@ -175,6 +175,13 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
-+ // Try adding localhost
-+ err = loc_writer_add_network(writer, &network, "::1/128");
-+ if (err != -EINVAL) {
-+ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
-+ exit(EXIT_FAILURE);
-+ }
-+
- FILE* f = tmpfile();
- if (!f) {
- fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
---
-2.20.1
-
-From fc1190aa11e3ff3d2dbf5f4d408c298e7916f46f Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 21 Oct 2020 13:44:50 +0000
-Subject: [PATCH 015/111] networks: Remove accidentially committed debug line
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 2 --
- 1 file changed, 2 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index c112a41..be88d75 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -208,8 +208,6 @@ FAIL:
- if (r)
- return r;
-
-- DEBUG(ctx, "GOT HERE\n");
--
- // Create a new network
- return loc_network_new(ctx, network, &first_address, prefix);
- }
---
-2.20.1
-
-From a1707d8983898b6878cdd5c68744bcc444e278ed Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 21 Oct 2020 13:53:36 +0000
-Subject: [PATCH 016/111] importer: Add search index to announcements table
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 1 +
- 1 file changed, 1 insertion(+)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index e87d378..d0fe5a6 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -152,6 +152,7 @@ class CLI(object):
- last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
- CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
- CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
-+ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
-
- -- autnums
- CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
---
-2.20.1
-
-From 991baf530d47adb2ed7a15b65dc4565d07fa6d07 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 21 Oct 2020 13:54:45 +0000
-Subject: [PATCH 017/111] importer: Add search index to network_overrides table
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 2 ++
- 1 file changed, 2 insertions(+)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index d0fe5a6..fe21d73 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -189,6 +189,8 @@ class CLI(object):
- );
- CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
- ON network_overrides(network);
-+ CREATE INDEX IF NOT EXISTS network_overrides_search
-+ ON network_overrides USING GIST(network inet_ops);
- """)
-
- return db
---
-2.20.1
-
-From bbea93a74651df10e2ffdbd09eb434dc6a0471bc Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 21 Oct 2020 16:01:57 +0000
-Subject: [PATCH 018/111] importer: Restructure SQL query to be executed in
- parallel
-
-There are no functional changes, this just runs quicker now.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 67 ++++++++++++++++++---------------
- 1 file changed, 37 insertions(+), 30 deletions(-)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index fe21d73..3c1e5e2 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -237,34 +237,24 @@ class CLI(object):
-
- # Select all known networks
- rows = self.db.query("""
-- -- Get a (sorted) list of all known networks
-- WITH known_networks AS (
-- SELECT network FROM announcements
-- UNION
-- SELECT network FROM networks
-- UNION
-- SELECT network FROM network_overrides
-- ORDER BY network
-- )
--
- -- Return a list of those networks enriched with all
- -- other information that we store in the database
- SELECT
-- DISTINCT ON (known_networks.network)
-- known_networks.network AS network,
-- announcements.autnum AS autnum,
-+ DISTINCT ON (network)
-+ network,
-+ autnum,
-
- -- Country
- COALESCE(
- (
- SELECT country FROM network_overrides overrides
-- WHERE announcements.network <<= overrides.network
-+ WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
- ),
- (
- SELECT country FROM autnum_overrides overrides
-- WHERE announcements.autnum = overrides.number
-+ WHERE networks.autnum = overrides.number
- ),
- networks.country
- ) AS country,
-@@ -273,50 +263,67 @@ class CLI(object):
- COALESCE(
- (
- SELECT is_anonymous_proxy FROM network_overrides overrides
-- WHERE announcements.network <<= overrides.network
-+ WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
- ),
- (
- SELECT is_anonymous_proxy FROM autnum_overrides overrides
-- WHERE announcements.autnum = overrides.number
-+ WHERE networks.autnum = overrides.number
- ),
- FALSE
- ) AS is_anonymous_proxy,
- COALESCE(
- (
- SELECT is_satellite_provider FROM network_overrides overrides
-- WHERE announcements.network <<= overrides.network
-+ WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
- ),
- (
- SELECT is_satellite_provider FROM autnum_overrides overrides
-- WHERE announcements.autnum = overrides.number
-+ WHERE networks.autnum = overrides.number
- ),
- FALSE
- ) AS is_satellite_provider,
- COALESCE(
- (
- SELECT is_anycast FROM network_overrides overrides
-- WHERE announcements.network <<= overrides.network
-+ WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
- ),
- (
- SELECT is_anycast FROM autnum_overrides overrides
-- WHERE announcements.autnum = overrides.number
-+ WHERE networks.autnum = overrides.number
- ),
- FALSE
-- ) AS is_anycast,
--
-- -- Must be part of returned values for ORDER BY clause
-- masklen(announcements.network) AS sort_a,
-- masklen(networks.network) AS sort_b
-- FROM known_networks
-- LEFT JOIN announcements ON known_networks.network <<= announcements.network
-- LEFT JOIN networks ON known_networks.network <<= networks.network
-- ORDER BY known_networks.network, sort_a DESC, sort_b DESC
-+ ) AS is_anycast
-+ FROM (
-+ SELECT
-+ known_networks.network AS network,
-+ announcements.autnum AS autnum,
-+ networks.country AS country,
-+
-+ -- Must be part of returned values for ORDER BY clause
-+ masklen(announcements.network) AS sort_a,
-+ masklen(networks.network) AS sort_b
-+ FROM (
-+ SELECT network FROM announcements
-+ UNION ALL
-+ SELECT network FROM networks
-+ UNION ALL
-+ SELECT network FROM network_overrides
-+ ) known_networks
-+ LEFT JOIN
-+ announcements ON known_networks.network <<= announcements.network
-+ LEFT JOIN
-+ networks ON known_networks.network <<= networks.network
-+ ORDER BY
-+ known_networks.network,
-+ sort_a DESC,
-+ sort_b DESC
-+ ) networks
- """)
-
- for row in rows:
---
-2.20.1
-
-From 26ab419b68d166f932db1f97c38cb9d793d04187 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 22 Oct 2020 12:24:34 +0000
-Subject: [PATCH 019/111] network: Allow adding single IP addresses and
- automatically add the prefix
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 33 +++++++++++++++------------------
- src/test-network.c | 4 ++--
- 2 files changed, 17 insertions(+), 20 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index be88d75..d7b1645 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -161,6 +161,7 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
- const char* address_string) {
- struct in6_addr first_address;
- char* prefix_string;
-+ unsigned int prefix = 128;
- int r = -EINVAL;
-
- DEBUG(ctx, "Attempting to parse network %s\n", address_string);
-@@ -174,21 +175,6 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
-
- DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
-
-- // We need to have a prefix
-- if (!prefix_string) {
-- DEBUG(ctx, "No prefix set\n");
-- goto FAIL;
-- }
--
-- // Convert prefix to integer
-- unsigned int prefix = strtol(prefix_string, NULL, 10);
--
-- // End if the prefix was invalid
-- if (!prefix) {
-- DEBUG(ctx, "The prefix is zero or not a number\n");
-- goto FAIL;
-- }
--
- // Parse the address
- r = loc_parse_address(ctx, address_string, &first_address);
- if (r) {
-@@ -196,9 +182,20 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
- goto FAIL;
- }
-
-- // Map the prefix to IPv6 if needed
-- if (IN6_IS_ADDR_V4MAPPED(&first_address))
-- prefix += 96;
-+ // If a prefix was given, we will try to parse it
-+ if (prefix_string) {
-+ // Convert prefix to integer
-+ prefix = strtol(prefix_string, NULL, 10);
-+
-+ if (!prefix) {
-+ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
-+ goto FAIL;
-+ }
-+
-+ // Map the prefix to IPv6 if needed
-+ if (IN6_IS_ADDR_V4MAPPED(&first_address))
-+ prefix += 96;
-+ }
-
- FAIL:
- // Free temporary buffer
-diff --git a/src/test-network.c b/src/test-network.c
-index 8c7e898..b6776b4 100644
---- a/src/test-network.c
-+++ b/src/test-network.c
-@@ -170,8 +170,8 @@ int main(int argc, char** argv) {
-
- // Try adding a single address
- err = loc_writer_add_network(writer, &network, "2001:db8::");
-- if (err != -EINVAL) {
-- fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
-+ if (err) {
-+ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
- exit(EXIT_FAILURE);
- }
-
---
-2.20.1
-
-From aadac4c569e921be1d28dd3b2377ac7f3732213e Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Wed, 21 Oct 2020 14:47:36 +0000
-Subject: [PATCH 020/111] Revert "Revert "Revert "Revert "importer: Import raw
- sources for inetnum's again""""
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-This reverts commit 44341478233115b26bb27fdb24da5b0a1eedb173.
-
-Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/importer.py | 14 ++++----
- src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
- 2 files changed, 70 insertions(+), 7 deletions(-)
-
-diff --git a/src/python/importer.py b/src/python/importer.py
-index de20f37..f19db4b 100644
---- a/src/python/importer.py
-+++ b/src/python/importer.py
-@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
- "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
-
- # Asia Pacific Network Information Centre
-- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
-+ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-+ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
- #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
- #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
- "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
-@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
- # XXX ???
-
- # Réseaux IP Européens
-- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
-+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
-@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
-
- EXTENDED_SOURCES = (
- # African Network Information Centre
-- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
-+ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
-
- # Asia Pacific Network Information Centre
-- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
-+ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
-
- # American Registry for Internet Numbers
- "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
-@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
-
- # Réseaux IP Européens
-- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
-+ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
- )
-
- class Downloader(object):
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index 3c1e5e2..e8a4fc5 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -405,6 +405,10 @@ class CLI(object):
- if line.startswith("aut-num:"):
- return self._parse_autnum_block(block)
-
-+ # inetnum
-+ if line.startswith("inet6num:") or line.startswith("inetnum:"):
-+ return self._parse_inetnum_block(block)
-+
- # organisation
- elif line.startswith("organisation:"):
- return self._parse_org_block(block)
-@@ -434,6 +438,65 @@ class CLI(object):
- autnum.get("asn"), autnum.get("org"),
- )
-
-+ def _parse_inetnum_block(self, block):
-+ logging.debug("Parsing inetnum block:")
-+
-+ inetnum = {}
-+ for line in block:
-+ logging.debug(line)
-+
-+ # Split line
-+ key, val = split_line(line)
-+
-+ if key == "inetnum":
-+ start_address, delim, end_address = val.partition("-")
-+
-+ # Strip any excess space
-+ start_address, end_address = start_address.rstrip(), end_address.strip()
-+
-+ # Convert to IP address
-+ try:
-+ start_address = ipaddress.ip_address(start_address)
-+ end_address = ipaddress.ip_address(end_address)
-+ except ValueError:
-+ logging.warning("Could not parse line: %s" % line)
-+ return
-+
-+ # Set prefix to default
-+ prefix = 32
-+
-+ # Count number of addresses in this subnet
-+ num_addresses = int(end_address) - int(start_address)
-+ if num_addresses:
-+ prefix -= math.log(num_addresses, 2)
-+
-+ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
-+
-+ elif key == "inet6num":
-+ inetnum[key] = val
-+
-+ elif key == "country":
-+ if val == "UNITED STATES":
-+ val = "US"
-+
-+ inetnum[key] = val.upper()
-+
-+ # Skip empty objects
-+ if not inetnum:
-+ return
-+
-+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-+
-+ # Bail out in case we have processed a non-public IP network
-+ if network.is_private:
-+ logging.warning("Skipping non-globally routable network: %s" % network)
-+ return
-+
-+ self.db.execute("INSERT INTO networks(network, country) \
-+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
-+ "%s" % network, inetnum.get("country"),
-+ )
-+
- def _parse_org_block(self, block):
- org = {}
- for line in block:
---
-2.20.1
-
-From 002deb6b42ac0b3624c07e3352cebd72dc0685a2 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Wed, 21 Oct 2020 14:47:37 +0000
-Subject: [PATCH 021/111] Revert "Revert "location-importer.in: only import
- relevant data from AFRINIC, APNIC and RIPE""
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-This reverts commit 13f67f285856e8eabfeff2daf1be3aeaa36a82cc.
-
-Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
- 1 file changed, 87 insertions(+), 2 deletions(-)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index e8a4fc5..5656c41 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -166,6 +166,7 @@ class CLI(object):
- -- networks
- CREATE TABLE IF NOT EXISTS networks(network inet, country text);
- CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
-+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
- CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
-
- -- overrides
-@@ -375,6 +376,16 @@ class CLI(object):
- CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
- ON COMMIT DROP;
- CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
-+
-+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
-+ ON COMMIT DROP;
-+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
-+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
-+ """)
-+
-+ # Remove all previously imported content
-+ self.db.execute("""
-+ TRUNCATE TABLE networks;
- """)
-
- for source in location.importer.WHOIS_SOURCES:
-@@ -382,6 +393,67 @@ class CLI(object):
- for block in f:
- self._parse_block(block)
-
-+ # Process all parsed networks from every RIR we happen to have access to,
-+ # insert the largest network chunks into the networks table immediately...
-+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
-+
-+ for family in (row.family for row in families):
-+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
-+
-+ self.db.execute("INSERT INTO networks(network, country) \
-+ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
-+
-+ # ... determine any other prefixes for this network family, ...
-+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
-+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
-+
-+ # ... and insert networks with this prefix in case they provide additional
-+ # information (i. e. subnet of a larger chunk with a different country)
-+ for prefix in (row.prefix for row in prefixes):
-+ self.db.execute("""
-+ WITH candidates AS (
-+ SELECT
-+ _rirdata.network,
-+ _rirdata.country
-+ FROM
-+ _rirdata
-+ WHERE
-+ family(_rirdata.network) = %s
-+ AND
-+ masklen(_rirdata.network) = %s
-+ ),
-+ filtered AS (
-+ SELECT
-+ DISTINCT ON (c.network)
-+ c.network,
-+ c.country,
-+ masklen(networks.network),
-+ networks.country AS parent_country
-+ FROM
-+ candidates c
-+ LEFT JOIN
-+ networks
-+ ON
-+ c.network << networks.network
-+ ORDER BY
-+ c.network,
-+ masklen(networks.network) DESC NULLS LAST
-+ )
-+ INSERT INTO
-+ networks(network, country)
-+ SELECT
-+ network,
-+ country
-+ FROM
-+ filtered
-+ WHERE
-+ parent_country IS NULL
-+ OR
-+ country <> parent_country
-+ ON CONFLICT DO NOTHING""",
-+ family, prefix,
-+ )
-+
- self.db.execute("""
- INSERT INTO autnums(number, name)
- SELECT _autnums.number, _organizations.name FROM _autnums
-@@ -482,17 +554,30 @@ class CLI(object):
- inetnum[key] = val.upper()
-
- # Skip empty objects
-- if not inetnum:
-+ if not inetnum or not "country" in inetnum:
- return
-
- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-
-+ # Bail out in case we have processed a network covering the entire IP range, which
-+ # is necessary to work around faulty (?) IPv6 network processing
-+ if network.prefixlen == 0:
-+ logging.warning("Skipping network covering the entire IP adress range: %s" % network)
-+ return
-+
-+ # Bail out in case we have processed a network whose prefix length indicates it is
-+ # not globally routable (we have decided not to process them at the moment, as they
-+ # significantly enlarge our database without providing very helpful additional information)
-+ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
-+ logging.info("Skipping network too small to be publicly announced: %s" % network)
-+ return
-+
- # Bail out in case we have processed a non-public IP network
- if network.is_private:
- logging.warning("Skipping non-globally routable network: %s" % network)
- return
-
-- self.db.execute("INSERT INTO networks(network, country) \
-+ self.db.execute("INSERT INTO _rirdata(network, country) \
- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
- "%s" % network, inetnum.get("country"),
- )
---
-2.20.1
-
-From 28c73fa3f4257e0a41e52af8a9643da414a6cb6f Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Wed, 21 Oct 2020 14:47:38 +0000
-Subject: [PATCH 022/111] export.py: fix exporting IP networks for crappy
- xt_geoip module
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-In contrast to the location database itself, the xt_geoip module
-consumes a list of IP networks for each country, and returns after the
-first match.
-
-We therefore need to...
-
-(a) sort IP networks by their size, allow as precise matches as possible
-(b) export _any_ IP networks - including inverted subnets - to prevent
- undefined overlaps
-(c) do the entire thing as fast as possible, consuming as less disk
- space as possible, which is why we can't just iterate over all /24
- chunks
-
-Partially fixes: #12499
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/export.py | 69 ++++++++++++++++++++++++++++++++++----------
- 1 file changed, 54 insertions(+), 15 deletions(-)
-
-diff --git a/src/python/export.py b/src/python/export.py
-index d15c6f0..5eaf43f 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -39,8 +39,8 @@ class OutputWriter(object):
- suffix = "networks"
- mode = "w"
-
-- def __init__(self, f, prefix=None, flatten=True):
-- self.f, self.prefix, self.flatten = f, prefix, flatten
-+ def __init__(self, db, f, prefix=None, flatten=True):
-+ self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten
-
- # The previously written network
- self._last_network = None
-@@ -49,13 +49,13 @@ class OutputWriter(object):
- self._write_header()
-
- @classmethod
-- def open(cls, filename, **kwargs):
-+ def open(cls, db, filename, **kwargs):
- """
- Convenience function to open a file
- """
- f = open(filename, cls.mode)
-
-- return cls(f, **kwargs)
-+ return cls(db, f, **kwargs)
-
- def __repr__(self):
- return "<%s f=%s>" % (self.__class__.__name__, self.f)
-@@ -87,13 +87,31 @@ class OutputWriter(object):
- def _write_network(self, network):
- self.f.write("%s\n" % network)
-
-- def write(self, network):
-+ def write(self, network, subnets):
- if self.flatten and self._flatten(network):
- log.debug("Skipping writing network %s" % network)
- return
-
-- # Write the network to file
-- self._write_network(network)
-+ # Write the network when it has no subnets
-+ if not subnets:
-+ network = ipaddress.ip_network("%s" % network)
-+ return self._write_network(network)
-+
-+ # Collect all matching subnets
-+ matching_subnets = []
-+
-+ for subnet in sorted(subnets):
-+ # Try to find the subnet in the database
-+ n = self.db.lookup("%s" % subnet.network_address)
-+
-+ # No entry or matching country means those IP addresses belong here
-+ if not n or n.country_code == network.country_code:
-+ matching_subnets.append(subnet)
-+
-+ # Write all networks as compact as possible
-+ for network in ipaddress.collapse_addresses(matching_subnets):
-+ log.debug("Writing %s to database" % network)
-+ self._write_network(network)
-
- def finish(self):
- """
-@@ -143,10 +161,10 @@ class XTGeoIPOutputWriter(OutputWriter):
- mode = "wb"
-
- def _write_network(self, network):
-- for address in (network.first_address, network.last_address):
-+ for address in (network.network_address, network.broadcast_address):
- # Convert this into a string of bits
- bytes = socket.inet_pton(
-- network.family, address,
-+ socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
- )
-
- self.f.write(bytes)
-@@ -175,7 +193,7 @@ class Exporter(object):
- directory, prefix=country_code, suffix=self.writer.suffix, family=family,
- )
-
-- writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
-+ writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code)
-
- # Create writers for ASNs
- for asn in asns:
-@@ -183,22 +201,43 @@ class Exporter(object):
- directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
- )
-
-- writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
-+ writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
-
- # Get all networks that match the family
- networks = self.db.search_networks(family=family)
-
-+ # Materialise the generator into a list (uses quite some memory)
-+ networks = list(networks)
-+
- # Walk through all networks
-- for network in networks:
-+ for i, network in enumerate(networks):
-+ _network = ipaddress.ip_network("%s" % network)
-+
-+ # Search for all subnets
-+ subnets = set()
-+
-+ while i < len(networks):
-+ subnet = networks[i+1]
-+
-+ if subnet.is_subnet_of(network):
-+ _subnet = ipaddress.ip_network("%s" % subnet)
-+
-+ subnets.add(_subnet)
-+ subnets.update(_network.address_exclude(_subnet))
-+
-+ i += 1
-+ else:
-+ break
-+
- # Write matching countries
- try:
-- writers[network.country_code].write(network)
-+ writers[network.country_code].write(network, subnets)
- except KeyError:
- pass
-
- # Write matching ASNs
- try:
-- writers[network.asn].write(network)
-+ writers[network.asn].write(network, subnets)
- except KeyError:
- pass
-
-@@ -209,7 +248,7 @@ class Exporter(object):
- country = flags[flag]
-
- try:
-- writers[country].write(network)
-+ writers[country].write(network, subnets)
- except KeyError:
- pass
-
---
-2.20.1
-
-From bd341642fc6bbcc050e9b4ec5124585c83cab84d Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Wed, 21 Oct 2020 14:47:39 +0000
-Subject: [PATCH 023/111] location-importer.in: filter bogus IP networks for
- both Whois and extended sources
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Sanity checks for parsed networks have been put into a separate function
-to avoid boilerplate code for extended sources. This makes the location
-database less vulnerable to garbage written into RIR databases on
-purpose or by chance.
-
-Fixes: #12500
-
-Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 83 ++++++++++++++++++++++++++-------
- 1 file changed, 67 insertions(+), 16 deletions(-)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index 5656c41..f24d357 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -469,6 +469,69 @@ class CLI(object):
- for line in f:
- self._parse_line(line)
-
-+ def _check_parsed_network(self, network):
-+ """
-+ Assistive function to detect and subsequently sort out parsed
-+ networks from RIR data (both Whois and so-called "extended sources"),
-+ which are or have...
-+
-+ (a) not globally routable (RFC 1918 space, et al.)
-+ (b) covering a too large chunk of the IP address space (prefix length
-+ is < 7 for IPv4 networks, and < 10 for IPv6)
-+ (c) "0.0.0.0" or "::" as a network address
-+ (d) are too small for being publicly announced (we have decided not to
-+ process them at the moment, as they significantly enlarge our
-+ database without providing very helpful additional information)
-+
-+ This unfortunately is necessary due to brain-dead clutter across
-+ various RIR databases, causing mismatches and eventually disruptions.
-+
-+ We will return False in case a network is not suitable for adding
-+ it to our database, and True otherwise.
-+ """
-+
-+ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
-+ return False
-+
-+ if not network.is_global:
-+ logging.warning("Skipping non-globally routable network: %s" % network)
-+ return False
-+
-+ if network.version == 4:
-+ if network.prefixlen < 7:
-+ logging.warning("Skipping too big IP chunk: %s" % network)
-+ return False
-+
-+ if network.prefixlen > 24:
-+ logging.info("Skipping network too small to be publicly announced: %s" % network)
-+ return False
-+
-+ if str(network.network_address) == "0.0.0.0":
-+ logging.warning("Skipping network based on 0.0.0.0: %s" % network)
-+ return False
-+
-+ elif network.version == 6:
-+ if network.prefixlen < 10:
-+ logging.warning("Skipping too big IP chunk: %s" % network)
-+ return False
-+
-+ if network.prefixlen > 48:
-+ logging.info("Skipping network too small to be publicly announced: %s" % network)
-+ return False
-+
-+ if str(network.network_address) == "::":
-+ logging.warning("Skipping network based on '::': %s" % network)
-+ return False
-+
-+ else:
-+ # This should not happen...
-+ logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
-+ return False
-+
-+ # In case we have made it here, the network is considered to
-+ # be suitable for libloc consumption...
-+ return True
-+
- def _parse_block(self, block):
- # Get first line to find out what type of block this is
- line = block[0]
-@@ -559,22 +622,7 @@ class CLI(object):
-
- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-
-- # Bail out in case we have processed a network covering the entire IP range, which
-- # is necessary to work around faulty (?) IPv6 network processing
-- if network.prefixlen == 0:
-- logging.warning("Skipping network covering the entire IP adress range: %s" % network)
-- return
--
-- # Bail out in case we have processed a network whose prefix length indicates it is
-- # not globally routable (we have decided not to process them at the moment, as they
-- # significantly enlarge our database without providing very helpful additional information)
-- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
-- logging.info("Skipping network too small to be publicly announced: %s" % network)
-- return
--
-- # Bail out in case we have processed a non-public IP network
-- if network.is_private:
-- logging.warning("Skipping non-globally routable network: %s" % network)
-+ if not self._check_parsed_network(network):
- return
-
- self.db.execute("INSERT INTO _rirdata(network, country) \
-@@ -658,6 +706,9 @@ class CLI(object):
- log.warning("Invalid IP address: %s" % address)
- return
-
-+ if not self._check_parsed_network(network):
-+ return
-+
- self.db.execute("INSERT INTO networks(network, country) \
- VALUES(%s, %s) ON CONFLICT (network) DO \
- UPDATE SET country = excluded.country",
---
-2.20.1
-
-From eee65490a10e0fe89b3834b8be176fc900084fa0 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Wed, 21 Oct 2020 14:47:40 +0000
-Subject: [PATCH 024/111] importer.py: fetch LACNIC data via HTTPS
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/importer.py | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/src/python/importer.py b/src/python/importer.py
-index f19db4b..5f46bc3 100644
---- a/src/python/importer.py
-+++ b/src/python/importer.py
-@@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
- "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
-
- # Latin America and Caribbean Network Information Centre
-- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
-+ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
-
- # Réseaux IP Européens
- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
---
-2.20.1
-
-From 84187ab5436eb158529d6f5e2a38890b4af3ddb4 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Wed, 21 Oct 2020 14:47:41 +0000
-Subject: [PATCH 025/111] location-importer.in: omit historic/orphaned RIR data
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Some RIRs include detailled information regarding networks not managed
-by or allocated to themselves, particually APNIC. We need to filter
-those networks (they usually have a characteristic network name) in
-order to prevent operational quirks or returning wrong country codes.
-
-Fixes: #12501
-Partially fixes: #12499
-
-Cc: Michael Tremer <michael.tremer@ipfire.org>
-Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 38 +++++++++++++++++++++------------
- 1 file changed, 24 insertions(+), 14 deletions(-)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index f24d357..a869256 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -494,38 +494,38 @@ class CLI(object):
- return False
-
- if not network.is_global:
-- logging.warning("Skipping non-globally routable network: %s" % network)
-+ log.warning("Skipping non-globally routable network: %s" % network)
- return False
-
- if network.version == 4:
- if network.prefixlen < 7:
-- logging.warning("Skipping too big IP chunk: %s" % network)
-+ log.warning("Skipping too big IP chunk: %s" % network)
- return False
-
- if network.prefixlen > 24:
-- logging.info("Skipping network too small to be publicly announced: %s" % network)
-+ log.info("Skipping network too small to be publicly announced: %s" % network)
- return False
-
- if str(network.network_address) == "0.0.0.0":
-- logging.warning("Skipping network based on 0.0.0.0: %s" % network)
-+ log.warning("Skipping network based on 0.0.0.0: %s" % network)
- return False
-
- elif network.version == 6:
- if network.prefixlen < 10:
-- logging.warning("Skipping too big IP chunk: %s" % network)
-+ log.warning("Skipping too big IP chunk: %s" % network)
- return False
-
- if network.prefixlen > 48:
-- logging.info("Skipping network too small to be publicly announced: %s" % network)
-+ log.info("Skipping network too small to be publicly announced: %s" % network)
- return False
-
- if str(network.network_address) == "::":
-- logging.warning("Skipping network based on '::': %s" % network)
-+ log.warning("Skipping network based on '::': %s" % network)
- return False
-
- else:
- # This should not happen...
-- logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
-+ log.warning("Skipping network of unknown family, this should not happen: %s" % network)
- return False
-
- # In case we have made it here, the network is considered to
-@@ -574,15 +574,22 @@ class CLI(object):
- )
-
- def _parse_inetnum_block(self, block):
-- logging.debug("Parsing inetnum block:")
-+ log.debug("Parsing inetnum block:")
-
- inetnum = {}
- for line in block:
-- logging.debug(line)
-+ log.debug(line)
-
- # Split line
- key, val = split_line(line)
-
-+ # Filter any inetnum records which are only referring to IP space
-+ # not managed by that specific RIR...
-+ if key == "netname":
-+ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
-+ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
-+ return
-+
- if key == "inetnum":
- start_address, delim, end_address = val.partition("-")
-
-@@ -594,7 +601,7 @@ class CLI(object):
- start_address = ipaddress.ip_address(start_address)
- end_address = ipaddress.ip_address(end_address)
- except ValueError:
-- logging.warning("Could not parse line: %s" % line)
-+ log.warning("Could not parse line: %s" % line)
- return
-
- # Set prefix to default
-@@ -611,15 +618,18 @@ class CLI(object):
- inetnum[key] = val
-
- elif key == "country":
-- if val == "UNITED STATES":
-- val = "US"
--
- inetnum[key] = val.upper()
-
- # Skip empty objects
- if not inetnum or not "country" in inetnum:
- return
-
-+ # Skip objects with bogus country code 'ZZ'
-+ if inetnum.get("country") == "ZZ":
-+ log.warning("Skipping network with bogus country 'ZZ': %s" % \
-+ (inetnum.get("inet6num") or inetnum.get("inetnum")))
-+ return
-+
- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-
- if not self._check_parsed_network(network):
---
-2.20.1
-
-From ebb087cfa30ec5ca0c96dcce66a91245c1ffc271 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Wed, 21 Oct 2020 14:47:43 +0000
-Subject: [PATCH 026/111] location-importer.in: avoid log spam for too small
- networks
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 4 ++--
- 1 file changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index a869256..864eab1 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -503,7 +503,7 @@ class CLI(object):
- return False
-
- if network.prefixlen > 24:
-- log.info("Skipping network too small to be publicly announced: %s" % network)
-+ log.debug("Skipping network too small to be publicly announced: %s" % network)
- return False
-
- if str(network.network_address) == "0.0.0.0":
-@@ -516,7 +516,7 @@ class CLI(object):
- return False
-
- if network.prefixlen > 48:
-- log.info("Skipping network too small to be publicly announced: %s" % network)
-+ log.debug("Skipping network too small to be publicly announced: %s" % network)
- return False
-
- if str(network.network_address) == "::":
---
-2.20.1
-
-From bbed1fd2330e8efa6b413dc152a1a6ef2d771aac Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 27 Oct 2020 17:14:30 +0000
-Subject: [PATCH 027/111] export: Flatten the tree before exporting it
-
-This patch removes the possibility that any IP address ranges
-might show up in multiple exported files.
-
-If this was content from the database:
-
- * 10.0.0.0/16 - DE
- * 10.0.1.0/24 - FR
-
-Then the IP address 10.0.1.1 would match for both countries.
-
-The algorithm will now break the larger /16 subnet apart into
-smaller subnets so that 10.0.1.0/24 is no longer overlapped.
-
-There was some time spent on this to make this as efficient
-as possible.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/export.py | 154 ++++++++++++++++++++++++++++++-------------
- 1 file changed, 110 insertions(+), 44 deletions(-)
-
-diff --git a/src/python/export.py b/src/python/export.py
-index 5eaf43f..dd44332 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -89,28 +89,55 @@ class OutputWriter(object):
-
- def write(self, network, subnets):
- if self.flatten and self._flatten(network):
-- log.debug("Skipping writing network %s" % network)
-+ log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
- return
-
-+ # Convert network into a Python object
-+ _network = ipaddress.ip_network("%s" % network)
-+
- # Write the network when it has no subnets
- if not subnets:
-- network = ipaddress.ip_network("%s" % network)
-- return self._write_network(network)
-+ log.debug("Writing %s to %s" % (_network, self.f))
-+ return self._write_network(_network)
-+
-+ # Convert subnets into Python objects
-+ _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
-+
-+ # Split the network into smaller bits so that
-+ # we can accomodate for any gaps in it later
-+ to_check = set()
-+ for _subnet in _subnets:
-+ to_check.update(
-+ _network.address_exclude(_subnet)
-+ )
-+
-+ # Clear the list of all subnets
-+ subnets = []
-+
-+ # Check if all subnets to not overlap with anything else
-+ while to_check:
-+ subnet_to_check = to_check.pop()
-
-- # Collect all matching subnets
-- matching_subnets = []
-+ for _subnet in _subnets:
-+ # Drop this subnet if it equals one of the subnets
-+ # or if it is subnet of one of them
-+ if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
-+ break
-
-- for subnet in sorted(subnets):
-- # Try to find the subnet in the database
-- n = self.db.lookup("%s" % subnet.network_address)
-+ # Break it down if it overlaps
-+ if subnet_to_check.overlaps(_subnet):
-+ to_check.update(
-+ subnet_to_check.address_exclude(_subnet)
-+ )
-+ break
-
-- # No entry or matching country means those IP addresses belong here
-- if not n or n.country_code == network.country_code:
-- matching_subnets.append(subnet)
-+ # Add the subnet again as it passed the check
-+ else:
-+ subnets.append(subnet_to_check)
-
- # Write all networks as compact as possible
-- for network in ipaddress.collapse_addresses(matching_subnets):
-- log.debug("Writing %s to database" % network)
-+ for network in ipaddress.collapse_addresses(subnets):
-+ log.debug("Writing %s to %s" % (network, self.f))
- self._write_network(network)
-
- def finish(self):
-@@ -206,40 +233,40 @@ class Exporter(object):
- # Get all networks that match the family
- networks = self.db.search_networks(family=family)
-
-- # Materialise the generator into a list (uses quite some memory)
-- networks = list(networks)
-+ # Create a stack with all networks in order where we can put items back
-+ # again and retrieve them in the next iteration.
-+ networks = BufferedStack(networks)
-
- # Walk through all networks
-- for i, network in enumerate(networks):
-- _network = ipaddress.ip_network("%s" % network)
--
-- # Search for all subnets
-- subnets = set()
--
-- while i < len(networks):
-- subnet = networks[i+1]
--
-- if subnet.is_subnet_of(network):
-- _subnet = ipaddress.ip_network("%s" % subnet)
--
-- subnets.add(_subnet)
-- subnets.update(_network.address_exclude(_subnet))
--
-- i += 1
-- else:
-+ for network in networks:
-+ # Collect all networks which are a subnet of network
-+ subnets = []
-+ for subnet in networks:
-+ # If the next subnet was not a subnet, we have to push
-+ # it back on the stack and break this loop
-+ if not subnet.is_subnet_of(network):
-+ networks.push(subnet)
- break
-
-+ subnets.append(subnet)
-+
- # Write matching countries
-- try:
-- writers[network.country_code].write(network, subnets)
-- except KeyError:
-- pass
-+ if network.country_code and network.country_code in writers:
-+ # Mismatching subnets
-+ gaps = [
-+ subnet for subnet in subnets if not network.country_code == subnet.country_code
-+ ]
-+
-+ writers[network.country_code].write(network, gaps)
-
- # Write matching ASNs
-- try:
-- writers[network.asn].write(network, subnets)
-- except KeyError:
-- pass
-+ if network.asn and network.asn in writers:
-+ # Mismatching subnets
-+ gaps = [
-+ subnet for subnet in subnets if not network.asn == subnet.asn
-+ ]
-+
-+ writers[network.asn].write(network, gaps)
-
- # Handle flags
- for flag in flags:
-@@ -247,10 +274,19 @@ class Exporter(object):
- # Fetch the "fake" country code
- country = flags[flag]
-
-- try:
-- writers[country].write(network, subnets)
-- except KeyError:
-- pass
-+ if not country in writers:
-+ continue
-+
-+ gaps = [
-+ subnet for subnet in subnets
-+ if not subnet.has_flag(flag)
-+ ]
-+
-+ writers[country].write(network, gaps)
-+
-+ # Push all subnets back onto the stack
-+ for subnet in reversed(subnets):
-+ networks.push(subnet)
-
- # Write everything to the filesystem
- for writer in writers.values():
-@@ -262,3 +298,33 @@ class Exporter(object):
- )
-
- return os.path.join(directory, filename)
-+
-+
-+class BufferedStack(object):
-+ """
-+ This class takes an iterator and when being iterated
-+ over it returns objects from that iterator for as long
-+ as there are any.
-+
-+ It additionally has a function to put an item back on
-+ the back so that it will be returned again at the next
-+ iteration.
-+ """
-+ def __init__(self, iterator):
-+ self.iterator = iterator
-+ self.stack = []
-+
-+ def __iter__(self):
-+ return self
-+
-+ def __next__(self):
-+ if self.stack:
-+ return self.stack.pop(0)
-+
-+ return next(self.iterator)
-+
-+ def push(self, elem):
-+ """
-+ Takes an element and puts it on the stack
-+ """
-+ self.stack.insert(0, elem)
---
-2.20.1
-
-From e99a72265c1ba2194b61663eda7e9f14e0083016 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 28 Oct 2020 09:52:36 +0000
-Subject: [PATCH 028/111] location: Fix Python syntax error in verify()
-
-The database is now being opened before the requested
-method is called and handle_verify() wasn't updated.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location.in | 8 +-------
- 1 file changed, 1 insertion(+), 7 deletions(-)
-
-diff --git a/src/python/location.in b/src/python/location.in
-index 44ad726..b5e5758 100644
---- a/src/python/location.in
-+++ b/src/python/location.in
-@@ -453,13 +453,7 @@ class CLI(object):
-
- return 0
-
-- def handle_verify(self, ns):
-- try:
-- db = location.Database(ns.database)
-- except FileNotFoundError as e:
-- log.error("%s: %s" % (ns.database, e))
-- return 127
--
-+ def handle_verify(self, db, ns):
- # Verify the database
- with open(ns.public_key, "r") as f:
- if not db.verify(f):
---
-2.20.1
-
-From 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Thu, 29 Oct 2020 07:25:53 -0700
-Subject: [PATCH 029/111] location update: Remove double conversion of
- timestamps
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-This caused that the timestamp in DNS was misinterpreted
-as local time and often, databases could not be downloaded.
-
-Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
----
- src/python/location.in | 5 +----
- 1 file changed, 1 insertion(+), 4 deletions(-)
-
-diff --git a/src/python/location.in b/src/python/location.in
-index b5e5758..070640c 100644
---- a/src/python/location.in
-+++ b/src/python/location.in
-@@ -421,11 +421,8 @@ class CLI(object):
- # Fetch the timestamp we need from DNS
- t = location.discover_latest_version()
-
-- # Parse timestamp into datetime format
-- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
--
- # Check the version of the local database
-- if db and timestamp and db.created_at >= timestamp.timestamp():
-+ if db and t and db.created_at >= t:
- log.info("Already on the latest version")
- return
-
---
-2.20.1
-
-From 60c1ac0307312614bd6980d30b44bb59b5a6ab6e Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Thu, 29 Oct 2020 07:36:46 -0700
-Subject: [PATCH 030/111] location.in: do not confuse UTC with local time zones
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
----
- src/python/location.in | 12 +++++-------
- 1 file changed, 5 insertions(+), 7 deletions(-)
-
-diff --git a/src/python/location.in b/src/python/location.in
-index 070640c..0d09210 100644
---- a/src/python/location.in
-+++ b/src/python/location.in
-@@ -398,10 +398,7 @@ class CLI(object):
-
- def handle_update(self, db, ns):
- if ns.cron and db:
-- now = datetime.datetime.utcnow()
--
-- # Parse the database timestamp
-- t = datetime.datetime.utcfromtimestamp(db.created_at)
-+ now = time.time()
-
- if ns.cron == "daily":
- delta = datetime.timedelta(days=1)
-@@ -410,11 +407,12 @@ class CLI(object):
- elif ns.cron == "monthly":
- delta = datetime.timedelta(days=30)
-
-+ delta = delta.total_seconds()
-+
- # Check if the database has recently been updated
-- if t >= (now - delta):
-+ if db.created_at >= (now - delta):
- log.info(
-- _("The database has been updated recently (%s)") % \
-- format_timedelta(now - t),
-+ _("The database has been updated recently"),
- )
- return 3
-
---
-2.20.1
-
-From e7d612e5219ef9ba612ed404e4e2c174110d3dd7 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
-Date: Tue, 3 Nov 2020 15:31:08 +0000
-Subject: [PATCH 031/111] location-importer.in: always convert organisation
- handles into upper cases
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Fixes: #12523
-
-Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location-importer.in | 6 ++++--
- 1 file changed, 4 insertions(+), 2 deletions(-)
-
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index 864eab1..2dec89e 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -560,7 +560,7 @@ class CLI(object):
- autnum["asn"] = m.group(2)
-
- elif key == "org":
-- autnum[key] = val
-+ autnum[key] = val.upper()
-
- # Skip empty objects
- if not autnum:
-@@ -646,7 +646,9 @@ class CLI(object):
- # Split line
- key, val = split_line(line)
-
-- if key in ("organisation", "org-name"):
-+ if key == "organisation":
-+ org[key] = val.upper()
-+ elif key == "org-name":
- org[key] = val
-
- # Skip empty objects
---
-2.20.1
-
-From e96704f43acca1a8f56d9a680cce281f5e587ec5 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 11 Nov 2020 21:16:45 +0000
-Subject: [PATCH 032/111] test: Add tests for database enumerator
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/test-database.c | 53 +++++++++++++++++++++++++++++++++++++++++++++
- 1 file changed, 53 insertions(+)
-
-diff --git a/src/test-database.c b/src/test-database.c
-index b4a75c4..4aef94e 100644
---- a/src/test-database.c
-+++ b/src/test-database.c
-@@ -38,6 +38,14 @@ const char* DESCRIPTION =
- "Maecenas ut venenatis nunc.";
- const char* LICENSE = "CC";
-
-+const char* networks[] = {
-+ "2001:db8::/32",
-+ "2001:db8:1000::/48",
-+ "2001:db8:2000::/48",
-+ "2001:db8:2020::/48",
-+ NULL,
-+};
-+
- static int attempt_to_open(struct loc_ctx* ctx, char* path) {
- FILE* f = fopen(path, "r");
- if (!f)
-@@ -139,6 +147,24 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
-+ struct loc_network* network = NULL;
-+
-+ // Add some networks
-+ const char** n = networks;
-+ while (*n) {
-+ err = loc_writer_add_network(writer, &network, *n);
-+ if (err) {
-+ fprintf(stderr, "Could not add network %s\n", *n);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Set a country
-+ loc_network_set_country_code(network, "XX");
-+
-+ // Next one
-+ n++;
-+ }
-+
- FILE* f = tmpfile();
- if (!f) {
- fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
-@@ -170,6 +196,33 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
-+ // Enumerator
-+ struct loc_database_enumerator* enumerator;
-+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
-+ if (err) {
-+ fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Walk through all networks
-+ while (1) {
-+ err = loc_database_enumerator_next_network(enumerator, &network);
-+ if (err) {
-+ fprintf(stderr, "Error fetching the next network: %d\n", err);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ if (!network)
-+ break;
-+
-+ char* s = loc_network_str(network);
-+ printf("Got network: %s\n", s);
-+ free(s);
-+ }
-+
-+ // Free the enumerator
-+ loc_database_enumerator_unref(enumerator);
-+
- // Close the database
- loc_database_unref(db);
- loc_unref(ctx);
---
-2.20.1
-
-From ecce288da39a2c0eb60076050ca21e9619f61844 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 11 Nov 2020 23:01:19 +0000
-Subject: [PATCH 033/111] networks: Add list to manage groups of networks
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 11 ++++++
- src/loc/network.h | 12 ++++++
- src/network.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++
- 3 files changed, 119 insertions(+)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index b8296eb..6ef2d27 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -98,6 +98,17 @@ global:
- loc_network_str;
- loc_network_unref;
-
-+ # Network List
-+ loc_network_list_clear;
-+ loc_network_list_empty;
-+ loc_network_list_get;
-+ loc_network_list_new;
-+ loc_network_list_pop;
-+ loc_network_list_push;
-+ loc_network_list_ref;
-+ loc_network_list_size;
-+ loc_network_list_unref;
-+
- # Writer
- loc_writer_add_as;
- loc_writer_add_country;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index 70c3803..fd20812 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -56,6 +56,18 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
-
- int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
-
-+// List
-+struct loc_network_list;
-+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
-+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
-+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
-+size_t loc_network_list_size(struct loc_network_list* list);
-+int loc_network_list_empty(struct loc_network_list* list);
-+void loc_network_list_clear(struct loc_network_list* list);
-+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
-+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
-+struct loc_network* loc_network_list_pop(struct loc_network_list* list);
-+
- #ifdef LIBLOC_PRIVATE
-
- int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
-diff --git a/src/network.c b/src/network.c
-index d7b1645..c9e7979 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -746,3 +746,99 @@ LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node)
- LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
- return loc_network_ref(node->network);
- }
-+
-+// List
-+
-+struct loc_network_list {
-+ struct loc_ctx* ctx;
-+ int refcount;
-+
-+ struct loc_network* list[1024];
-+ size_t size;
-+ size_t max_size;
-+};
-+
-+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
-+ struct loc_network_list** list) {
-+ struct loc_network_list* l = calloc(1, sizeof(*l));
-+ if (!l)
-+ return -ENOMEM;
-+
-+ l->ctx = loc_ref(ctx);
-+ l->refcount = 1;
-+
-+ // Do not allow this list to grow larger than this
-+ l->max_size = 1024;
-+
-+ DEBUG(l->ctx, "Network list allocated at %p\n", l);
-+ *list = l;
-+ return 0;
-+}
-+
-+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
-+ list->refcount++;
-+
-+ return list;
-+}
-+
-+static void loc_network_list_free(struct loc_network_list* list) {
-+ DEBUG(list->ctx, "Releasing network list at %p\n", list);
-+
-+ for (unsigned int i = 0; i < list->size; i++)
-+ loc_network_unref(list->list[i]);
-+
-+ loc_unref(list->ctx);
-+ free(list);
-+}
-+
-+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
-+ if (!list)
-+ return NULL;
-+
-+ if (--list->refcount > 0)
-+ return list;
-+
-+ loc_network_list_free(list);
-+ return NULL;
-+}
-+
-+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
-+ return list->size;
-+}
-+
-+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
-+ return list->size == 0;
-+}
-+
-+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
-+ for (unsigned int i = 0; i < list->size; i++)
-+ loc_network_unref(list->list[i]);
-+
-+ list->size = 0;
-+}
-+
-+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
-+ // Check index
-+ if (index >= list->size)
-+ return NULL;
-+
-+ return loc_network_ref(list->list[index]);
-+}
-+
-+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
-+ // Check if we have space left
-+ if (list->size == list->max_size)
-+ return -ENOMEM;
-+
-+ list->list[list->size++] = loc_network_ref(network);
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
-+ // Return nothing when empty
-+ if (loc_network_list_empty(list))
-+ return NULL;
-+
-+ return list->list[list->size--];
-+}
---
-2.20.1
-
-From 8b2205272b7872a1458ad87811abf58609f38ad4 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 13:57:35 +0000
-Subject: [PATCH 034/111] networks: Add function to dump lists
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 1 +
- src/loc/network.h | 1 +
- src/network.c | 14 ++++++++++++++
- 3 files changed, 16 insertions(+)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index 6ef2d27..a5641c6 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -100,6 +100,7 @@ global:
-
- # Network List
- loc_network_list_clear;
-+ loc_network_list_dump;
- loc_network_list_empty;
- loc_network_list_get;
- loc_network_list_new;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index fd20812..44c50a4 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -64,6 +64,7 @@ struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
- size_t loc_network_list_size(struct loc_network_list* list);
- int loc_network_list_empty(struct loc_network_list* list);
- void loc_network_list_clear(struct loc_network_list* list);
-+void loc_network_list_dump(struct loc_network_list* list);
- struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
- int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
- struct loc_network* loc_network_list_pop(struct loc_network_list* list);
-diff --git a/src/network.c b/src/network.c
-index c9e7979..0977406 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -817,6 +817,20 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
- list->size = 0;
- }
-
-+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
-+ struct loc_network* network;
-+ char* s;
-+
-+ for (unsigned int i = 0; i < list->size; i++) {
-+ network = list->list[i];
-+
-+ s = loc_network_str(network);
-+
-+ INFO(list->ctx, "%s\n", s);
-+ free(s);
-+ }
-+}
-+
- LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
- // Check index
- if (index >= list->size)
---
-2.20.1
-
-From 850e75167e8e03fe8b951992c9f7bc2ccb9fb711 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 14:18:40 +0000
-Subject: [PATCH 035/111] network: Add functions to break network into subnets
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 5 +
- src/loc/network.h | 6 +
- src/network.c | 290 ++++++++++++++++++++++++++++++++++++++++++++-
- src/test-network.c | 61 ++++++++++
- 4 files changed, 361 insertions(+), 1 deletion(-)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index a5641c6..fcb9ea5 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -80,6 +80,8 @@ global:
-
- # Network
- loc_network_address_family;
-+ loc_network_eq;
-+ loc_network_exclude;
- loc_network_format_first_address;
- loc_network_format_last_address;
- loc_network_get_asn;
-@@ -96,6 +98,7 @@ global:
- loc_network_set_country_code;
- loc_network_set_flag;
- loc_network_str;
-+ loc_network_subnets;
- loc_network_unref;
-
- # Network List
-@@ -107,7 +110,9 @@ global:
- loc_network_list_pop;
- loc_network_list_push;
- loc_network_list_ref;
-+ loc_network_list_reverse;
- loc_network_list_size;
-+ loc_network_list_sort;
- loc_network_list_unref;
-
- # Writer
-diff --git a/src/loc/network.h b/src/loc/network.h
-index 44c50a4..4e51a62 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -54,7 +54,11 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
- int loc_network_set_flag(struct loc_network* network, uint32_t flag);
- int loc_network_match_flag(struct loc_network* network, uint32_t flag);
-
-+int loc_network_eq(struct loc_network* self, struct loc_network* other);
- int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
-+struct loc_network_list* loc_network_subnets(struct loc_network* network);
-+struct loc_network_list* loc_network_exclude(
-+ struct loc_network* self, struct loc_network* other);
-
- // List
- struct loc_network_list;
-@@ -68,6 +72,8 @@ void loc_network_list_dump(struct loc_network_list* list);
- struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
- int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
- struct loc_network* loc_network_list_pop(struct loc_network_list* list);
-+void loc_network_list_sort(struct loc_network_list* list);
-+void loc_network_list_reverse(struct loc_network_list* list);
-
- #ifdef LIBLOC_PRIVATE
-
-diff --git a/src/network.c b/src/network.c
-index 0977406..6c08070 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -97,6 +97,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
- return a;
- }
-
-+static struct in6_addr address_increment(const struct in6_addr* address) {
-+ struct in6_addr a = *address;
-+
-+ for (int octet = 15; octet >= 0; octet--) {
-+ if (a.s6_addr[octet] < 255) {
-+ a.s6_addr[octet]++;
-+ break;
-+ } else {
-+ a.s6_addr[octet] = 0;
-+ }
-+ }
-+
-+ return a;
-+}
-+
- LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
- struct in6_addr* address, unsigned int prefix) {
- // Address cannot be unspecified
-@@ -405,6 +420,69 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
- return loc_network_has_flag(network, flag);
- }
-
-+LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
-+#ifdef ENABLE_DEBUG
-+ char* n1 = loc_network_str(self);
-+ char* n2 = loc_network_str(other);
-+
-+ DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
-+
-+ free(n1);
-+ free(n2);
-+#endif
-+
-+ // Family must be the same
-+ if (self->family != other->family) {
-+ DEBUG(self->ctx, " Family mismatch\n");
-+
-+ return 0;
-+ }
-+
-+ // The start address must be the same
-+ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
-+ DEBUG(self->ctx, " Address mismatch\n");
-+
-+ return 0;
-+ }
-+
-+ // The prefix length must be the same
-+ if (self->prefix != other->prefix) {
-+ DEBUG(self->ctx, " Prefix mismatch\n");
-+ return 0;
-+ }
-+
-+ DEBUG(self->ctx, " Yes!\n");
-+
-+ return 1;
-+}
-+
-+static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
-+ // Families must match
-+ if (self->family != other->family)
-+ return -1;
-+
-+ int r = in6_addr_cmp(&self->first_address, &other->first_address);
-+
-+ switch (r) {
-+ // Smaller
-+ case -1:
-+ return 0;
-+
-+ // Larger
-+ case 1:
-+ return 1;
-+
-+ default:
-+ break;
-+ }
-+
-+ if (self->prefix > other->prefix)
-+ return 1;
-+
-+ // Dunno
-+ return 0;
-+}
-+
- LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
- // If the start address of the other network is smaller than this network,
- // it cannot be a subnet.
-@@ -419,6 +497,175 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net
- return 1;
- }
-
-+LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
-+ struct loc_network_list* list;
-+
-+ // New prefix length
-+ unsigned int prefix = network->prefix + 1;
-+
-+ // Check if the new prefix is valid
-+ if (valid_prefix(&network->first_address, prefix))
-+ return NULL;
-+
-+ // Create a new list with the result
-+ int r = loc_network_list_new(network->ctx, &list);
-+ if (r) {
-+ ERROR(network->ctx, "Could not create network list: %d\n", r);
-+ return NULL;
-+ }
-+
-+ struct loc_network* subnet1 = NULL;
-+ struct loc_network* subnet2 = NULL;
-+
-+ // Create the first half of the network
-+ r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix);
-+ if (r)
-+ goto ERROR;
-+
-+ // The next subnet starts after the first one
-+ struct in6_addr first_address = address_increment(&subnet1->last_address);
-+
-+ // Create the second half of the network
-+ r = loc_network_new(network->ctx, &subnet2, &first_address, prefix);
-+ if (r)
-+ goto ERROR;
-+
-+ // Push the both onto the stack (in reverse order)
-+ r = loc_network_list_push(list, subnet2);
-+ if (r)
-+ goto ERROR;
-+
-+ r = loc_network_list_push(list, subnet1);
-+ if (r)
-+ goto ERROR;
-+
-+ loc_network_unref(subnet1);
-+ loc_network_unref(subnet2);
-+
-+ return list;
-+
-+ERROR:
-+ if (subnet1)
-+ loc_network_unref(subnet1);
-+
-+ if (subnet2)
-+ loc_network_unref(subnet2);
-+
-+ if (list)
-+ loc_network_list_unref(list);
-+
-+ return NULL;
-+}
-+
-+LOC_EXPORT struct loc_network_list* loc_network_exclude(
-+ struct loc_network* self, struct loc_network* other) {
-+ struct loc_network_list* list;
-+
-+#ifdef ENABLE_DEBUG
-+ char* n1 = loc_network_str(self);
-+ char* n2 = loc_network_str(other);
-+
-+ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
-+
-+ free(n1);
-+ free(n2);
-+#endif
-+
-+ // Family must match
-+ if (self->family != other->family) {
-+ DEBUG(self->ctx, "Family mismatch\n");
-+
-+ return NULL;
-+ }
-+
-+ // Other must be a subnet of self
-+ if (!loc_network_is_subnet_of(other, self)) {
-+ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
-+
-+ return NULL;
-+ }
-+
-+ // We cannot perform this operation if both networks equal
-+ if (loc_network_eq(self, other)) {
-+ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
-+
-+ return NULL;
-+ }
-+
-+ // Create a new list with the result
-+ int r = loc_network_list_new(self->ctx, &list);
-+ if (r) {
-+ ERROR(self->ctx, "Could not create network list: %d\n", r);
-+ return NULL;
-+ }
-+
-+ struct loc_network_list* subnets = loc_network_subnets(self);
-+
-+ struct loc_network* subnet1 = NULL;
-+ struct loc_network* subnet2 = NULL;
-+
-+ while (subnets) {
-+ // Fetch both subnets
-+ subnet1 = loc_network_list_get(subnets, 0);
-+ subnet2 = loc_network_list_get(subnets, 1);
-+
-+ // Free list
-+ loc_network_list_unref(subnets);
-+ subnets = NULL;
-+
-+ if (loc_network_eq(other, subnet1)) {
-+ r = loc_network_list_push(list, subnet2);
-+ if (r)
-+ goto ERROR;
-+
-+ } else if (loc_network_eq(other, subnet2)) {
-+ r = loc_network_list_push(list, subnet1);
-+ if (r)
-+ goto ERROR;
-+
-+ } else if (loc_network_is_subnet_of(other, subnet1)) {
-+ r = loc_network_list_push(list, subnet2);
-+ if (r)
-+ goto ERROR;
-+
-+ subnets = loc_network_subnets(subnet1);
-+
-+ } else if (loc_network_is_subnet_of(other, subnet2)) {
-+ r = loc_network_list_push(list, subnet1);
-+ if (r)
-+ goto ERROR;
-+
-+ subnets = loc_network_subnets(subnet2);
-+
-+ } else {
-+ ERROR(self->ctx, "We should never get here\n");
-+ goto ERROR;
-+ }
-+
-+ loc_network_unref(subnet1);
-+ loc_network_unref(subnet2);
-+ }
-+
-+#ifdef ENABLE_DEBUG
-+ loc_network_list_dump(list);
-+#endif
-+
-+ // Return the result
-+ return list;
-+
-+ERROR:
-+ if (subnet1)
-+ loc_network_unref(subnet1);
-+
-+ if (subnet2)
-+ loc_network_unref(subnet2);
-+
-+ if (list)
-+ loc_network_list_unref(list);
-+
-+ return NULL;
-+}
-+
- LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
- // Add country code
- loc_country_code_copy(dbobj->country_code, network->country_code);
-@@ -854,5 +1101,46 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
- if (loc_network_list_empty(list))
- return NULL;
-
-- return list->list[list->size--];
-+ return list->list[--list->size];
-+}
-+
-+static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
-+ // Do nothing for invalid indices
-+ if (i1 >= list->size || i2 >= list->size)
-+ return;
-+
-+ DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
-+
-+ struct loc_network* network1 = list->list[i1];
-+ struct loc_network* network2 = list->list[i2];
-+
-+ list->list[i1] = network2;
-+ list->list[i2] = network1;
-+}
-+
-+LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
-+ unsigned int i = 0;
-+ unsigned int j = list->size - 1;
-+
-+ while (i < j) {
-+ loc_network_list_swap(list, i++, j--);
-+ }
-+}
-+
-+LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
-+ unsigned int n = list->size;
-+ int swapped;
-+
-+ do {
-+ swapped = 0;
-+
-+ for (unsigned int i = 1; i < n; i++) {
-+ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
-+ loc_network_list_swap(list, i-1, i);
-+ swapped = 1;
-+ }
-+ }
-+
-+ n--;
-+ } while (swapped);
- }
-diff --git a/src/test-network.c b/src/test-network.c
-index b6776b4..af1b2e6 100644
---- a/src/test-network.c
-+++ b/src/test-network.c
-@@ -118,6 +118,19 @@ int main(int argc, char** argv) {
- size_t nodes = loc_network_tree_count_nodes(tree);
- printf("The tree has %zu nodes\n", nodes);
-
-+ // Check equals function
-+ err = loc_network_eq(network1, network1);
-+ if (!err) {
-+ fprintf(stderr, "Network is not equal with itself\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ err = loc_network_eq(network1, network2);
-+ if (err) {
-+ fprintf(stderr, "Networks equal unexpectedly\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
- // Check subnet function
- err = loc_network_is_subnet_of(network1, network2);
- if (err != 0) {
-@@ -131,6 +144,54 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
-+ // Make list of subnets
-+ struct loc_network_list* subnets = loc_network_subnets(network1);
-+ if (!subnets) {
-+ fprintf(stderr, "Could not find subnets of network\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ loc_network_list_dump(subnets);
-+
-+ while (!loc_network_list_empty(subnets)) {
-+ struct loc_network* subnet = loc_network_list_pop(subnets);
-+ if (!subnet) {
-+ fprintf(stderr, "Received an empty subnet\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ char* s = loc_network_str(subnet);
-+ printf("Received subnet %s\n", s);
-+ free(s);
-+
-+ if (!loc_network_is_subnet_of(subnet, network1)) {
-+ fprintf(stderr, "Not a subnet\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ loc_network_unref(subnet);
-+ }
-+
-+ loc_network_list_unref(subnets);
-+
-+ struct loc_network_list* excluded = loc_network_exclude(network1, network2);
-+ if (!excluded) {
-+ fprintf(stderr, "Could not create excluded list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ loc_network_list_dump(excluded);
-+
-+ // Reverse it
-+ loc_network_list_reverse(excluded);
-+ loc_network_list_dump(excluded);
-+
-+ // Sort them and dump them again
-+ loc_network_list_sort(excluded);
-+ loc_network_list_dump(excluded);
-+
-+ loc_network_list_unref(excluded);
-+
- // Create a database
- struct loc_writer* writer;
- err = loc_writer_new(ctx, &writer, NULL, NULL);
---
-2.20.1
-
-From 6159d384c4a98fe45ec52522e2950719e4982d80 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 14:24:58 +0000
-Subject: [PATCH 036/111] networks: Add function to check if two networks
- overlap
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 1 +
- src/loc/network.h | 1 +
- src/network.c | 16 ++++++++++++++++
- 3 files changed, 18 insertions(+)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index fcb9ea5..0c2835b 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -93,6 +93,7 @@ global:
- loc_network_match_flag;
- loc_network_new;
- loc_network_new_from_string;
-+ loc_network_overlaps;
- loc_network_ref;
- loc_network_set_asn;
- loc_network_set_country_code;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index 4e51a62..ef13756 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -55,6 +55,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
- int loc_network_match_flag(struct loc_network* network, uint32_t flag);
-
- int loc_network_eq(struct loc_network* self, struct loc_network* other);
-+int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
- int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
- struct loc_network_list* loc_network_subnets(struct loc_network* network);
- struct loc_network_list* loc_network_exclude(
-diff --git a/src/network.c b/src/network.c
-index 6c08070..d826511 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -483,6 +483,22 @@ static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
- return 0;
- }
-
-+LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
-+ if (loc_network_match_address(self, &other->first_address) == 0)
-+ return 1;
-+
-+ if (loc_network_match_address(self, &other->last_address) == 0)
-+ return 1;
-+
-+ if (loc_network_match_address(other, &self->first_address) == 0)
-+ return 1;
-+
-+ if (loc_network_match_address(other, &self->last_address) == 0)
-+ return 1;
-+
-+ return 0;
-+}
-+
- LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
- // If the start address of the other network is smaller than this network,
- // it cannot be a subnet.
---
-2.20.1
-
-From e52ba21761f27e592040a2793b2a26bbeeeecc05 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 14:28:15 +0000
-Subject: [PATCH 037/111] networks: Add function to check if network is part of
- a list
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 1 +
- src/loc/network.h | 1 +
- src/network.c | 9 +++++++++
- 3 files changed, 11 insertions(+)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index 0c2835b..f1b63a2 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -104,6 +104,7 @@ global:
-
- # Network List
- loc_network_list_clear;
-+ loc_network_list_contains;
- loc_network_list_dump;
- loc_network_list_empty;
- loc_network_list_get;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index ef13756..7804512 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -73,6 +73,7 @@ void loc_network_list_dump(struct loc_network_list* list);
- struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
- int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
- struct loc_network* loc_network_list_pop(struct loc_network_list* list);
-+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
- void loc_network_list_sort(struct loc_network_list* list);
- void loc_network_list_reverse(struct loc_network_list* list);
-
-diff --git a/src/network.c b/src/network.c
-index d826511..fcbdc59 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -1120,6 +1120,15 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
- return list->list[--list->size];
- }
-
-+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
-+ for (unsigned int i = 0; i < list->size; i++) {
-+ if (loc_network_eq(list->list[i], network))
-+ return 1;
-+ }
-+
-+ return 0;
-+}
-+
- static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
- // Do nothing for invalid indices
- if (i1 >= list->size || i2 >= list->size)
---
-2.20.1
-
-From f802f3a4decf4827ecc8bcabe269ae9f94f7f32d Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 14:33:22 +0000
-Subject: [PATCH 038/111] networks: Add function to merge two lists
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 1 +
- src/loc/network.h | 1 +
- src/network.c | 13 +++++++++++++
- 3 files changed, 15 insertions(+)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index f1b63a2..c0b6b1f 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -108,6 +108,7 @@ global:
- loc_network_list_dump;
- loc_network_list_empty;
- loc_network_list_get;
-+ loc_network_list_merge;
- loc_network_list_new;
- loc_network_list_pop;
- loc_network_list_push;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index 7804512..e30d91c 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -76,6 +76,7 @@ struct loc_network* loc_network_list_pop(struct loc_network_list* list);
- int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
- void loc_network_list_sort(struct loc_network_list* list);
- void loc_network_list_reverse(struct loc_network_list* list);
-+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
-
- #ifdef LIBLOC_PRIVATE
-
-diff --git a/src/network.c b/src/network.c
-index fcbdc59..541286d 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -1169,3 +1169,16 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
- n--;
- } while (swapped);
- }
-+
-+LOC_EXPORT int loc_network_list_merge(
-+ struct loc_network_list* self, struct loc_network_list* other) {
-+ int r;
-+
-+ for (unsigned int i = 0; i < other->size; i++) {
-+ r = loc_network_list_push(self, other->list[i]);
-+ if (r)
-+ return r;
-+ }
-+
-+ return 0;
-+}
---
-2.20.1
-
-From 6d22a179dffd08fcf2a44aafb361725ab22486d4 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 14:35:43 +0000
-Subject: [PATCH 039/111] network: Make lists unique
-
-Networks that are in the list won't be added again
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 4 ++++
- 1 file changed, 4 insertions(+)
-
-diff --git a/src/network.c b/src/network.c
-index 541286d..44571b3 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -1103,6 +1103,10 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
- }
-
- LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
-+ // Do not add networks that are already on the list
-+ if (loc_network_list_contains(list, network))
-+ return 0;
-+
- // Check if we have space left
- if (list->size == list->max_size)
- return -ENOMEM;
---
-2.20.1
-
-From 681ff05cb7cdf230d38abf09a330a31498e265a4 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 19:21:13 +0000
-Subject: [PATCH 040/111] database: Pass flag to enumerator to flatten output
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 8 +++++++-
- src/loc/database.h | 6 +++++-
- src/perl/Location.xs | 2 +-
- src/python/database.c | 25 ++++++++++++++++++-------
- 4 files changed, 31 insertions(+), 10 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index fa1dad0..9baab33 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -104,6 +104,9 @@ struct loc_database_enumerator {
- enum loc_network_flags flags;
- int family;
-
-+ // Flatten output?
-+ int flatten;
-+
- // Index of the AS we are looking at
- unsigned int as_index;
-
-@@ -933,7 +936,7 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
- // Enumerator
-
- LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
-- struct loc_database* db, enum loc_database_enumerator_mode mode) {
-+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
- struct loc_database_enumerator* e = calloc(1, sizeof(*e));
- if (!e)
- return -ENOMEM;
-@@ -944,6 +947,9 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
- e->mode = mode;
- e->refcount = 1;
-
-+ // Flatten output?
-+ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
-+
- // Initialise graph search
- //e->network_stack[++e->network_stack_depth] = 0;
- e->network_stack_depth = 1;
-diff --git a/src/loc/database.h b/src/loc/database.h
-index 43173dd..14eb5ea 100644
---- a/src/loc/database.h
-+++ b/src/loc/database.h
-@@ -55,9 +55,13 @@ enum loc_database_enumerator_mode {
- LOC_DB_ENUMERATE_COUNTRIES = 3,
- };
-
-+enum loc_database_enumerator_flags {
-+ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
-+};
-+
- struct loc_database_enumerator;
- int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
-- struct loc_database* db, enum loc_database_enumerator_mode mode);
-+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
- struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
- struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
-
-diff --git a/src/perl/Location.xs b/src/perl/Location.xs
-index dcf3f0d..b7676d2 100644
---- a/src/perl/Location.xs
-+++ b/src/perl/Location.xs
-@@ -125,7 +125,7 @@ database_countries(db)
- PPCODE:
- // Create Database enumerator
- struct loc_database_enumerator* enumerator;
-- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
-+ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
-
- if (err) {
- croak("Could not create a database enumerator\n");
-diff --git a/src/python/database.c b/src/python/database.c
-index 1013a58..7f8c2c2 100644
---- a/src/python/database.c
-+++ b/src/python/database.c
-@@ -207,10 +207,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
- return (PyObject*)self;
- }
-
--static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
-+static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
- struct loc_database_enumerator* enumerator;
-
-- int r = loc_database_enumerator_new(&enumerator, self->db, what);
-+ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
- if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
- return NULL;
-@@ -223,7 +223,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
- }
-
- static PyObject* Database_ases(DatabaseObject* self) {
-- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
-+ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
- }
-
- static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
-@@ -234,7 +234,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
-
- struct loc_database_enumerator* enumerator;
-
-- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
-+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
- if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
- return NULL;
-@@ -250,7 +250,11 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
- }
-
- static PyObject* Database_networks(DatabaseObject* self) {
-- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
-+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
-+}
-+
-+static PyObject* Database_networks_flattened(DatabaseObject *self) {
-+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
- }
-
- static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
-@@ -264,7 +268,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
- return NULL;
-
- struct loc_database_enumerator* enumerator;
-- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
-+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
- if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
- return NULL;
-@@ -317,7 +321,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
- }
-
- static PyObject* Database_countries(DatabaseObject* self) {
-- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
-+ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
- }
-
- static struct PyMethodDef Database_methods[] = {
-@@ -403,6 +407,13 @@ static struct PyGetSetDef Database_getsetters[] = {
- NULL,
- NULL,
- },
-+ {
-+ "networks_flattened",
-+ (getter)Database_networks_flattened,
-+ NULL,
-+ NULL,
-+ NULL,
-+ },
- {
- "vendor",
- (getter)Database_get_vendor,
---
-2.20.1
-
-From f5e50a47e37e9b29d0d2ee9e5a41e5a5fe5aea7f Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 19:21:58 +0000
-Subject: [PATCH 041/111] network: Reduce debugging output
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 28 +++-------------------------
- 1 file changed, 3 insertions(+), 25 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index 44571b3..f7071a6 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -421,37 +421,17 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
- }
-
- LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
--#ifdef ENABLE_DEBUG
-- char* n1 = loc_network_str(self);
-- char* n2 = loc_network_str(other);
--
-- DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
--
-- free(n1);
-- free(n2);
--#endif
--
- // Family must be the same
-- if (self->family != other->family) {
-- DEBUG(self->ctx, " Family mismatch\n");
--
-+ if (self->family != other->family)
- return 0;
-- }
-
- // The start address must be the same
-- if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
-- DEBUG(self->ctx, " Address mismatch\n");
--
-+ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0)
- return 0;
-- }
-
- // The prefix length must be the same
-- if (self->prefix != other->prefix) {
-- DEBUG(self->ctx, " Prefix mismatch\n");
-+ if (self->prefix != other->prefix)
- return 0;
-- }
--
-- DEBUG(self->ctx, " Yes!\n");
-
- return 1;
- }
-@@ -1138,8 +1118,6 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
- if (i1 >= list->size || i2 >= list->size)
- return;
-
-- DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
--
- struct loc_network* network1 = list->list[i1];
- struct loc_network* network2 = list->list[i2];
-
---
-2.20.1
-
-From 037c65d3a07ec6d37ff063f0645adda6b483b407 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 19:36:38 +0000
-Subject: [PATCH 042/111] python: Export networks exclude function
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/network.c | 39 +++++++++++++++++++++++++++++++++++++++
- 1 file changed, 39 insertions(+)
-
-diff --git a/src/python/network.c b/src/python/network.c
-index 5496d1e..11f672b 100644
---- a/src/python/network.c
-+++ b/src/python/network.c
-@@ -24,6 +24,24 @@
- #include "locationmodule.h"
- #include "network.h"
-
-+static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
-+ PyObject* list = PyList_New(0);
-+ if (!networks)
-+ return list;
-+
-+ while (!loc_network_list_empty(networks)) {
-+ struct loc_network* network = loc_network_list_pop(networks);
-+
-+ PyObject* n = new_network(&NetworkType, network);
-+ PyList_Append(list, n);
-+
-+ loc_network_unref(network);
-+ Py_DECREF(n);
-+ }
-+
-+ return list;
-+}
-+
- PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
- NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
- if (self) {
-@@ -154,6 +172,21 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
- Py_RETURN_NONE;
- }
-
-+static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
-+ NetworkObject* other = NULL;
-+
-+ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
-+ return NULL;
-+
-+ struct loc_network_list* list = loc_network_exclude(self->network, other->network);
-+
-+ // Convert to Python objects
-+ PyObject* obj = PyList_FromNetworkList(list);
-+ loc_network_list_unref(list);
-+
-+ return obj;
-+}
-+
- static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
- NetworkObject* other = NULL;
-
-@@ -191,6 +224,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
- }
-
- static struct PyMethodDef Network_methods[] = {
-+ {
-+ "exclude",
-+ (PyCFunction)Network_exclude,
-+ METH_VARARGS,
-+ NULL,
-+ },
- {
- "has_flag",
- (PyCFunction)Network_has_flag,
---
-2.20.1
-
-From 9a7732c8679e805d4d2d55ea4750c5d70ca4bd2c Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 19:59:22 +0000
-Subject: [PATCH 043/111] network: Add more debugging output to stacks
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 16 +++++++++++++---
- 1 file changed, 13 insertions(+), 3 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index f7071a6..d41e873 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -1088,8 +1088,12 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
- return 0;
-
- // Check if we have space left
-- if (list->size == list->max_size)
-+ if (list->size == list->max_size) {
-+ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
- return -ENOMEM;
-+ }
-+
-+ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
-
- list->list[list->size++] = loc_network_ref(network);
-
-@@ -1098,10 +1102,16 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
-
- LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
- // Return nothing when empty
-- if (loc_network_list_empty(list))
-+ if (loc_network_list_empty(list)) {
-+ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
- return NULL;
-+ }
-
-- return list->list[--list->size];
-+ struct loc_network* network = list->list[--list->size];
-+
-+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
-+
-+ return network;
- }
-
- LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
---
-2.20.1
-
-From 33a051e0435f6e78cc936f26f3b9ee16b7851025 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 20:00:09 +0000
-Subject: [PATCH 044/111] network: Add new subnet function
-
-The old one is too difficult to use in terms of order
-of input parameters and return value.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 1 +
- src/loc/network.h | 1 +
- src/network.c | 15 +++++++++++++++
- 3 files changed, 17 insertions(+)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index c0b6b1f..5392437 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -87,6 +87,7 @@ global:
- loc_network_get_asn;
- loc_network_get_country_code;
- loc_network_has_flag;
-+ loc_network_is_subnet;
- loc_network_is_subnet_of;
- loc_network_match_asn;
- loc_network_match_country_code;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index e30d91c..2154cdc 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -56,6 +56,7 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
-
- int loc_network_eq(struct loc_network* self, struct loc_network* other);
- int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
-+int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
- int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
- struct loc_network_list* loc_network_subnets(struct loc_network* network);
- struct loc_network_list* loc_network_exclude(
-diff --git a/src/network.c b/src/network.c
-index d41e873..5719111 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -479,6 +479,21 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
- return 0;
- }
-
-+LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
-+ // If the start address of the other network is smaller than this network,
-+ // it cannot be a subnet.
-+ if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
-+ return 0;
-+
-+ // If the end address of the other network is greater than this network,
-+ // it cannot be a subnet.
-+ if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
-+ return 0;
-+
-+ return 1;
-+}
-+
-+// XXX DEPRECATED - I find this too difficult to use
- LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
- // If the start address of the other network is smaller than this network,
- // it cannot be a subnet.
---
-2.20.1
-
-From add5bb652ba1dad1127f79cb6a0db2d363a6d5e5 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 20:01:17 +0000
-Subject: [PATCH 045/111] network: Add function to exclude multiple networks at
- once
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 1 +
- src/loc/network.h | 2 ++
- src/network.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++
- 3 files changed, 88 insertions(+)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index 5392437..bcd11be 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -82,6 +82,7 @@ global:
- loc_network_address_family;
- loc_network_eq;
- loc_network_exclude;
-+ loc_network_exclude_list;
- loc_network_format_first_address;
- loc_network_format_last_address;
- loc_network_get_asn;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index 2154cdc..40712b9 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -61,6 +61,8 @@ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other
- struct loc_network_list* loc_network_subnets(struct loc_network* network);
- struct loc_network_list* loc_network_exclude(
- struct loc_network* self, struct loc_network* other);
-+struct loc_network_list* loc_network_exclude_list(
-+ struct loc_network* network, struct loc_network_list* list);
-
- // List
- struct loc_network_list;
-diff --git a/src/network.c b/src/network.c
-index 5719111..751e8e5 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -677,6 +677,91 @@ ERROR:
- return NULL;
- }
-
-+LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
-+ struct loc_network* network, struct loc_network_list* list) {
-+ struct loc_network_list* to_check;
-+
-+ // Create a new list with all networks to look at
-+ int r = loc_network_list_new(network->ctx, &to_check);
-+ if (r)
-+ return NULL;
-+
-+ struct loc_network* subnet = NULL;
-+ struct loc_network_list* subnets = NULL;
-+
-+ for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
-+ subnet = loc_network_list_get(list, i);
-+
-+ // Find all excluded networks
-+ struct loc_network_list* excluded = loc_network_exclude(network, subnet);
-+ if (excluded) {
-+ // Add them all to the "to check" list
-+ loc_network_list_merge(to_check, excluded);
-+ loc_network_list_unref(excluded);
-+ }
-+
-+ // Cleanup
-+ loc_network_unref(subnet);
-+ }
-+
-+ r = loc_network_list_new(network->ctx, &subnets);
-+ if (r) {
-+ loc_network_list_unref(to_check);
-+ return NULL;
-+ }
-+
-+ while (!loc_network_list_empty(to_check)) {
-+ struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
-+
-+ // Marks whether this subnet passed all checks
-+ int passed = 1;
-+
-+ for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
-+ subnet = loc_network_list_get(list, i);
-+
-+ // Drop this subnet if is is already in list
-+ if (loc_network_eq(subnet_to_check, subnet)) {
-+ passed = 0;
-+ loc_network_unref(subnet);
-+ break;
-+ }
-+
-+ // Drop this subnet if is a subnet of another subnet
-+ if (loc_network_is_subnet_of(subnet, subnet_to_check)) {
-+ passed = 0;
-+ loc_network_unref(subnet);
-+ break;
-+ }
-+
-+ // Break it down if it overlaps
-+ if (loc_network_overlaps(subnet_to_check, subnet)) {
-+ passed = 0;
-+
-+ struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet);
-+ if (excluded) {
-+ loc_network_list_merge(to_check, excluded);
-+ loc_network_list_unref(excluded);
-+ }
-+
-+ loc_network_unref(subnet);
-+ break;
-+ }
-+
-+ loc_network_unref(subnet);
-+ }
-+
-+ if (passed) {
-+ r = loc_network_list_push(subnets, subnet_to_check);
-+ }
-+
-+ loc_network_unref(subnet_to_check);
-+ }
-+
-+ loc_network_list_unref(to_check);
-+
-+ return subnets;
-+}
-+
- LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
- // Add country code
- loc_country_code_copy(dbobj->country_code, network->country_code);
---
-2.20.1
-
-From d87fd7a3d277b4b03222c7d1680e51b3e45e525b Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 20:02:03 +0000
-Subject: [PATCH 046/111] database: Add option to return networks flattened
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 174 ++++++++++++++++++++++++++++++++++++++++---------
- 1 file changed, 143 insertions(+), 31 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index 9baab33..7a3d1a7 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -118,6 +118,9 @@ struct loc_database_enumerator {
- struct loc_node_stack network_stack[MAX_STACK_DEPTH];
- int network_stack_depth;
- unsigned int* networks_visited;
-+
-+ // For subnet search
-+ struct loc_network_list* stack;
- };
-
- static int loc_database_read_magic(struct loc_database* db) {
-@@ -935,6 +938,26 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
-
- // Enumerator
-
-+static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
-+ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
-+
-+ // Release all references
-+ loc_database_unref(enumerator->db);
-+ loc_unref(enumerator->ctx);
+diff --git a/src/country-list.c b/src/country-list.c
+new file mode 100644
+index 0000000..cc36740
+--- /dev/null
++++ b/src/country-list.c
+@@ -0,0 +1,161 @@
++/*
++ libloc - A library to determine the location of someone on the Internet
+
-+ if (enumerator->string)
-+ free(enumerator->string);
++ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
+
-+ // Free network search
-+ free(enumerator->networks_visited);
++ This library is free software; you can redistribute it and/or
++ modify it under the terms of the GNU Lesser General Public
++ License as published by the Free Software Foundation; either
++ version 2.1 of the License, or (at your option) any later version.
+
-+ // Free subnet stack
-+ if (enumerator->stack)
-+ loc_network_list_unref(enumerator->stack);
++ This library is distributed in the hope that it will be useful,
++ but WITHOUT ANY WARRANTY; without even the implied warranty of
++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ Lesser General Public License for more details.
++*/
+
-+ free(enumerator);
-+}
++#include <errno.h>
++#include <stdlib.h>
+
- LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
- struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
- struct loc_database_enumerator* e = calloc(1, sizeof(*e));
-@@ -951,10 +974,16 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
- e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
-
- // Initialise graph search
-- //e->network_stack[++e->network_stack_depth] = 0;
- e->network_stack_depth = 1;
- e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
-
-+ // Allocate stack
-+ int r = loc_network_list_new(e->ctx, &e->stack);
-+ if (r) {
-+ loc_database_enumerator_free(e);
-+ return r;
-+ }
++#include <loc/country.h>
++#include <loc/country-list.h>
++#include <loc/private.h>
+
- DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
-
- *enumerator = e;
-@@ -967,22 +996,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
- return enumerator;
- }
-
--static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
-- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
--
-- // Release all references
-- loc_database_unref(enumerator->db);
-- loc_unref(enumerator->ctx);
--
-- if (enumerator->string)
-- free(enumerator->string);
--
-- // Free network search
-- free(enumerator->networks_visited);
--
-- free(enumerator);
--}
--
- LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
- if (!enumerator)
- return NULL;
-@@ -1116,17 +1129,13 @@ static int loc_database_enumerator_stack_push_node(
- return 0;
- }
-
--LOC_EXPORT int loc_database_enumerator_next_network(
-- struct loc_database_enumerator* enumerator, struct loc_network** network) {
-- // Reset network
-- *network = NULL;
--
-- // Do not do anything if not in network mode
-- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
-+static int __loc_database_enumerator_next_network(
-+ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
-+ // Return top element from the stack
-+ *network = loc_network_list_pop(enumerator->stack);
-+ if (*network)
- return 0;
-
-- int r;
--
- DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
- enumerator->network_stack_depth);
-
-@@ -1155,7 +1164,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
- enumerator->db->network_nodes_v1 + node->offset;
-
- // Add edges to stack
-- r = loc_database_enumerator_stack_push_node(enumerator,
-+ int r = loc_database_enumerator_stack_push_node(enumerator,
- be32toh(n->one), 1, node->depth + 1);
-
- if (r)
-@@ -1181,6 +1190,10 @@ LOC_EXPORT int loc_database_enumerator_next_network(
- if (r)
- return r;
-
-+ // Return all networks when the filter is disabled
-+ if (!filter)
-+ return 0;
++struct loc_country_list {
++ struct loc_ctx* ctx;
++ int refcount;
+
- // Check if we are interested in this network
-
- // Skip if the family does not match
-@@ -1223,12 +1236,111 @@ LOC_EXPORT int loc_database_enumerator_next_network(
- }
-
- // Reached the end of the search
-+ return 0;
-+}
-
-- // Mark all nodes as non-visited
-- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
-- enumerator->networks_visited[i] = 0;
-+static int __loc_database_enumerator_next_network_flattened(
-+ struct loc_database_enumerator* enumerator, struct loc_network** network) {
-+ // Fetch the next network
-+ int r = __loc_database_enumerator_next_network(enumerator, network, 1);
-+ if (r)
-+ return r;
-
-- return 0;
-+ // End if we could not read another network
-+ if (!*network)
-+ return 0;
++ struct loc_country** elements;
++ size_t elements_size;
+
-+ struct loc_network* subnet = NULL;
-+ struct loc_network_list* subnets;
++ size_t size;
++};
+
-+ // Create a list with all subnets
-+ r = loc_network_list_new(enumerator->ctx, &subnets);
-+ if (r)
-+ return r;
++static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
++ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
++ list, size, list->elements_size + size);
+
-+ // Search all subnets from the database
-+ while (1) {
-+ // Fetch the next network in line
-+ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
-+ if (r)
-+ goto END;
++ struct loc_country** elements = reallocarray(list->elements,
++ list->elements_size + size, sizeof(*list->elements));
++ if (!elements)
++ return -errno;
+
-+ // End if we did not receive another subnet
-+ if (!subnet)
-+ break;
++ list->elements = elements;
++ list->elements_size += size;
+
-+ // Collect all subnets in a list
-+ if (loc_network_is_subnet(*network, subnet)) {
-+ r = loc_network_list_push(subnets, subnet);
-+ if (r)
-+ goto END;
++ return 0;
++}
+
-+ loc_network_unref(subnet);
-+ continue;
-+ }
++LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
++ struct loc_country_list** list) {
++ struct loc_country_list* l = calloc(1, sizeof(*l));
++ if (!l)
++ return -ENOMEM;
+
-+ // If this is not a subnet, we push it back onto the stack and break
-+ r = loc_network_list_push(enumerator->stack, subnet);
-+ if (r)
-+ goto END;
++ l->ctx = loc_ref(ctx);
++ l->refcount = 1;
+
-+ loc_network_unref(subnet);
-+ break;
-+ }
++ DEBUG(l->ctx, "Country list allocated at %p\n", l);
++ *list = l;
+
-+ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
++ return 0;
++}
+
-+ // We can abort here if the network has no subnets
-+ if (loc_network_list_empty(subnets)) {
-+ loc_network_list_unref(subnets);
++LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
++ list->refcount++;
+
-+ return 0;
-+ }
++ return list;
++}
+
-+ // If the network has any subnets, we will break it into smaller parts
-+ // without the subnets.
-+ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
-+ if (!excluded || loc_network_list_empty(excluded)) {
-+ r = 1;
-+ goto END;
-+ }
++static void loc_country_list_free(struct loc_country_list* list) {
++ DEBUG(list->ctx, "Releasing country list at %p\n", list);
+
-+ // Sort the result
-+ loc_network_list_sort(excluded);
++ loc_country_list_clear(list);
+
-+ // Reverse the list
-+ loc_network_list_reverse(excluded);
++ loc_unref(list->ctx);
++ free(list);
++}
+
-+ // Replace network with the first one
-+ loc_network_unref(*network);
++LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
++ if (!list)
++ return NULL;
+
-+ *network = loc_network_list_pop(excluded);
++ if (--list->refcount > 0)
++ return list;
+
-+ // Push the rest onto the stack
-+ loc_network_list_merge(enumerator->stack, excluded);
++ loc_country_list_free(list);
++ return NULL;
++}
+
-+ loc_network_list_unref(excluded);
++LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
++ return list->size;
++}
+
-+END:
-+ if (subnet)
-+ loc_network_unref(subnet);
++LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
++ return list->size == 0;
++}
+
-+ loc_network_list_unref(subnets);
++LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
++ if (!list->elements)
++ return;
+
-+ return r;
++ for (unsigned int i = 0; i < list->size; i++)
++ loc_country_unref(list->elements[i]);
++
++ free(list->elements);
++ list->elements = NULL;
++ list->elements_size = 0;
++
++ list->size = 0;
+}
+
-+LOC_EXPORT int loc_database_enumerator_next_network(
-+ struct loc_database_enumerator* enumerator, struct loc_network** network) {
-+ // Do not do anything if not in network mode
-+ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
-+ return 0;
++LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
++ // Check index
++ if (index >= list->size)
++ return NULL;
+
-+ // Flatten output?
-+ if (enumerator->flatten)
-+ return __loc_database_enumerator_next_network_flattened(enumerator, network);
++ return loc_country_ref(list->elements[index]);
++}
+
-+ return __loc_database_enumerator_next_network(enumerator, network, 1);
- }
-
- LOC_EXPORT int loc_database_enumerator_next_country(
---
-2.20.1
-
-From d3ae93c27dcd7f6984fdc29cc141621e277f2e2a Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 20:09:20 +0000
-Subject: [PATCH 047/111] test: Update API
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/test-as.c | 2 +-
- src/test-database.c | 2 +-
- 2 files changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/src/test-as.c b/src/test-as.c
-index 839a04c..2d61675 100644
---- a/src/test-as.c
-+++ b/src/test-as.c
-@@ -95,7 +95,7 @@ int main(int argc, char** argv) {
- // Enumerator
-
- struct loc_database_enumerator* enumerator;
-- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
-+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
- if (err) {
- fprintf(stderr, "Could not create a database enumerator\n");
- exit(EXIT_FAILURE);
-diff --git a/src/test-database.c b/src/test-database.c
-index 4aef94e..da4b11c 100644
---- a/src/test-database.c
-+++ b/src/test-database.c
-@@ -198,7 +198,7 @@ int main(int argc, char** argv) {
-
- // Enumerator
- struct loc_database_enumerator* enumerator;
-- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
-+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
- if (err) {
- fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
- exit(EXIT_FAILURE);
---
-2.20.1
-
-From 594ca328c6e124d0f1eb543e9c8d9bbfe8a7b628 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 12 Nov 2020 20:09:37 +0000
-Subject: [PATCH 048/111] networks: Copy all attributes when splitting networks
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 14 ++++++++++++++
- 1 file changed, 14 insertions(+)
-
-diff --git a/src/network.c b/src/network.c
-index 751e8e5..d67f116 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -550,6 +550,20 @@ LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* netw
- if (r)
- goto ERROR;
-
-+ // Copy country code
-+ const char* country_code = loc_network_get_country_code(network);
-+ if (country_code) {
-+ loc_network_set_country_code(subnet1, country_code);
-+ loc_network_set_country_code(subnet2, country_code);
-+ }
++LOC_EXPORT int loc_country_list_append(
++ struct loc_country_list* list, struct loc_country* country) {
++ if (loc_country_list_contains(list, country))
++ return 0;
+
-+ // Copy ASN
-+ uint32_t asn = loc_network_get_asn(network);
-+ if (asn) {
-+ loc_network_set_asn(subnet1, asn);
-+ loc_network_set_asn(subnet2, asn);
++ // Check if we have space left
++ if (list->size >= list->elements_size) {
++ int r = loc_country_list_grow(list, 64);
++ if (r)
++ return r;
+ }
+
- loc_network_unref(subnet1);
- loc_network_unref(subnet2);
-
---
-2.20.1
-
-From 69248038292e9ea1a4ee8912cdfc8700456753ad Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 13 Nov 2020 11:23:33 +0000
-Subject: [PATCH 049/111] database: Move network filtering into a separate
- function
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 56 +++++++++++++++++++++++---------------------------
- 1 file changed, 26 insertions(+), 30 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index 7a3d1a7..72bc8eb 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1129,6 +1129,31 @@ static int loc_database_enumerator_stack_push_node(
- return 0;
- }
-
-+static int loc_database_enumerator_filter_network(
-+ struct loc_database_enumerator* enumerator, struct loc_network* network) {
-+ // Skip if the family does not match
-+ if (enumerator->family && loc_network_address_family(network) != enumerator->family)
-+ return 1;
++ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
+
-+ // Skip if the country code does not match
-+ if (*enumerator->country_code &&
-+ !loc_network_match_country_code(network, enumerator->country_code))
-+ return 1;
++ list->elements[list->size++] = loc_country_ref(country);
+
-+ // Skip if the ASN does not match
-+ if (enumerator->asn &&
-+ !loc_network_match_asn(network, enumerator->asn))
-+ return 1;
++ return 0;
++}
+
-+ // Skip if flags do not match
-+ if (enumerator->flags &&
-+ !loc_network_match_flag(network, enumerator->flags))
-+ return 1;
++LOC_EXPORT int loc_country_list_contains(
++ struct loc_country_list* list, struct loc_country* country) {
++ for (unsigned int i = 0; i < list->size; i++) {
++ if (loc_country_cmp(country, list->elements[i]) == 0)
++ return 1;
++ }
+
-+ // Do not filter
+ return 0;
+}
+
- static int __loc_database_enumerator_next_network(
- struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
- // Return top element from the stack
-@@ -1195,36 +1220,7 @@ static int __loc_database_enumerator_next_network(
- return 0;
-
- // Check if we are interested in this network
--
-- // Skip if the family does not match
-- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
-- loc_network_unref(*network);
-- *network = NULL;
--
-- continue;
-- }
--
-- // Skip if the country code does not match
-- if (*enumerator->country_code &&
-- !loc_network_match_country_code(*network, enumerator->country_code)) {
-- loc_network_unref(*network);
-- *network = NULL;
--
-- continue;
-- }
--
-- // Skip if the ASN does not match
-- if (enumerator->asn &&
-- !loc_network_match_asn(*network, enumerator->asn)) {
-- loc_network_unref(*network);
-- *network = NULL;
--
-- continue;
-- }
--
-- // Skip if flags do not match
-- if (enumerator->flags &&
-- !loc_network_match_flag(*network, enumerator->flags)) {
-+ if (loc_database_enumerator_filter_network(enumerator, *network)) {
- loc_network_unref(*network);
- *network = NULL;
-
---
-2.20.1
-
-From 2113e71bf7b997c82670c5c22cf91aa6442fe6f3 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 13 Nov 2020 11:29:02 +0000
-Subject: [PATCH 050/111] database: Filter results coming from stack
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 18 ++++++++++++++++--
- 1 file changed, 16 insertions(+), 2 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index 72bc8eb..0f3cdc2 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1157,9 +1157,23 @@ static int loc_database_enumerator_filter_network(
- static int __loc_database_enumerator_next_network(
- struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
- // Return top element from the stack
-- *network = loc_network_list_pop(enumerator->stack);
-- if (*network)
-+ while (1) {
-+ *network = loc_network_list_pop(enumerator->stack);
++LOC_EXPORT int loc_country_list_contains_code(
++ struct loc_country_list* list, const char* code) {
++ struct loc_country* country;
+
-+ // Stack is empty
-+ if (!*network)
-+ break;
++ int r = loc_country_new(list->ctx, &country, code);
++ if (r)
++ return -1;
+
-+ // Throw away any networks by filter
-+ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
-+ loc_network_unref(*network);
-+ *network = NULL;
-+ continue;
-+ }
++ r = loc_country_list_contains(list, country);
++ loc_country_unref(country);
+
-+ // Return result
- return 0;
-+ }
++ return r;
++}
+diff --git a/src/country.c b/src/country.c
+index 2ba93e6..7aac0db 100644
+--- a/src/country.c
++++ b/src/country.c
+@@ -34,6 +34,9 @@ struct loc_country {
+ };
- DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
- enumerator->network_stack_depth);
---
-2.20.1
-
-From d33753688138c9938743dafbbdddf220dd2afd14 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 13 Nov 2020 11:29:15 +0000
-Subject: [PATCH 051/111] network: Sort result of excluded lists
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 3 ---
- src/network.c | 3 +++
- 2 files changed, 3 insertions(+), 3 deletions(-)
-
+ LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
++ if (!loc_country_code_is_valid(country_code))
++ return -EINVAL;
++
+ struct loc_country* c = calloc(1, sizeof(*c));
+ if (!c)
+ return -ENOMEM;
diff --git a/src/database.c b/src/database.c
-index 0f3cdc2..6849d97 100644
+index fa1dad0..1871b74 100644
--- a/src/database.c
+++ b/src/database.c
-@@ -1315,9 +1315,6 @@ static int __loc_database_enumerator_next_network_flattened(
- goto END;
- }
+@@ -38,8 +38,10 @@
-- // Sort the result
-- loc_network_list_sort(excluded);
--
- // Reverse the list
- loc_network_list_reverse(excluded);
+ #include <loc/libloc.h>
+ #include <loc/as.h>
++#include <loc/as-list.h>
+ #include <loc/compat.h>
+ #include <loc/country.h>
++#include <loc/country-list.h>
+ #include <loc/database.h>
+ #include <loc/format.h>
+ #include <loc/network.h>
+@@ -99,11 +101,14 @@ struct loc_database_enumerator {
-diff --git a/src/network.c b/src/network.c
-index d67f116..9d02bf8 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -773,6 +773,9 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
+ // Search string
+ char* string;
+- char country_code[3];
+- uint32_t asn;
++ struct loc_country_list* countries;
++ struct loc_as_list* asns;
+ enum loc_network_flags flags;
+ int family;
- loc_network_list_unref(to_check);
++ // Flatten output?
++ int flatten;
++
+ // Index of the AS we are looking at
+ unsigned int as_index;
-+ // Sort the result
-+ loc_network_list_sort(subnets);
+@@ -115,6 +120,9 @@ struct loc_database_enumerator {
+ struct loc_node_stack network_stack[MAX_STACK_DEPTH];
+ int network_stack_depth;
+ unsigned int* networks_visited;
+
- return subnets;
- }
++ // For subnet search
++ struct loc_network_list* stack;
+ };
---
-2.20.1
-
-From 8d777f12f7ffa4df1b28d197563888296803b727 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 13 Nov 2020 11:38:15 +0000
-Subject: [PATCH 052/111] network: Add function to pop first element from stack
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 6 ++----
- src/libloc.sym | 1 +
- src/loc/network.h | 1 +
- src/network.c | 19 +++++++++++++++++++
- 4 files changed, 23 insertions(+), 4 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index 6849d97..b9d870f 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1315,15 +1315,13 @@ static int __loc_database_enumerator_next_network_flattened(
- goto END;
+ static int loc_database_read_magic(struct loc_database* db) {
+@@ -611,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
}
-- // Reverse the list
-- loc_network_list_reverse(excluded);
--
- // Replace network with the first one
- loc_network_unref(*network);
+ clock_t end = clock();
+- DEBUG(db->ctx, "Signature checked in %.4fms\n",
++ INFO(db->ctx, "Signature checked in %.4fms\n",
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
-- *network = loc_network_list_pop(excluded);
-+ *network = loc_network_list_pop_first(excluded);
+ CLEANUP:
+@@ -671,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
+ off_t lo = 0;
+ off_t hi = db->as_count - 1;
- // Push the rest onto the stack
-+ loc_network_list_reverse(excluded);
- loc_network_list_merge(enumerator->stack, excluded);
++#ifdef ENABLE_DEBUG
+ // Save start time
+ clock_t start = clock();
++#endif
- loc_network_list_unref(excluded);
-diff --git a/src/libloc.sym b/src/libloc.sym
-index bcd11be..6139db6 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -113,6 +113,7 @@ global:
- loc_network_list_merge;
- loc_network_list_new;
- loc_network_list_pop;
-+ loc_network_list_pop_first;
- loc_network_list_push;
- loc_network_list_ref;
- loc_network_list_reverse;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index 40712b9..203e61c 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -76,6 +76,7 @@ void loc_network_list_dump(struct loc_network_list* list);
- struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
- int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
- struct loc_network* loc_network_list_pop(struct loc_network_list* list);
-+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
- int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
- void loc_network_list_sort(struct loc_network_list* list);
- void loc_network_list_reverse(struct loc_network_list* list);
-diff --git a/src/network.c b/src/network.c
-index 9d02bf8..e7dc97e 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -1231,6 +1231,25 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
- return network;
- }
+ while (lo <= hi) {
+ off_t i = (lo + hi) / 2;
+@@ -685,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
+ // Check if this is a match
+ uint32_t as_number = loc_as_get_number(*as);
+ if (as_number == number) {
++#ifdef ENABLE_DEBUG
+ clock_t end = clock();
-+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
-+ // Return nothing when empty
-+ if (loc_network_list_empty(list)) {
-+ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
-+ return NULL;
-+ }
-+
-+ struct loc_network* network = list->list[0];
-+
-+ // Move all elements to the top of the stack
-+ for (unsigned int i = 0; i < --list->size; i++) {
-+ list->list[i] = list->list[i+1];
-+ }
-+
-+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
-+
-+ return network;
-+}
-+
- LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
- for (unsigned int i = 0; i < list->size; i++) {
- if (loc_network_eq(list->list[i], network))
---
-2.20.1
-
-From 7933f5bfb4dd7603cb646e192840762bf6394292 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 13 Nov 2020 11:43:53 +0000
-Subject: [PATCH 053/111] network: Unexport all tree functions
-
-These should not be exported
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 28 ++++++++++++++--------------
- src/test-network.c | 9 +++++++++
- 2 files changed, 23 insertions(+), 14 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index e7dc97e..d015579 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -847,7 +847,7 @@ struct loc_network_tree_node {
- struct loc_network* network;
- };
+ // Log how fast this has been
+ DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
++#endif
--LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
-+int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
- struct loc_network_tree* t = calloc(1, sizeof(*t));
- if (!t)
- return -ENOMEM;
-@@ -867,7 +867,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
- return 0;
- }
+ return 0;
+ }
+@@ -733,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
+ return -1;
+ }
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
-+struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
- return loc_network_tree_node_ref(tree->root);
- }
++#ifdef ENABLE_DEBUG
+ if (r == 0) {
+ char* string = loc_network_str(*network);
+ DEBUG(db->ctx, "Got network %s\n", string);
+ free(string);
+ }
++#endif
-@@ -939,7 +939,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
- return 0;
+ return r;
}
+@@ -762,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru
+ }
--LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
-+int loc_network_tree_walk(struct loc_network_tree* tree,
- int(*filter_callback)(struct loc_network* network, void* data),
- int(*callback)(struct loc_network* network, void* data), void* data) {
- return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
-@@ -954,7 +954,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
- free(tree);
- }
+ // Check if the given IP address is inside the network
+- r = loc_network_match_address(*network, address);
+- if (r) {
++ if (!loc_network_match_address(*network, address)) {
+ DEBUG(db->ctx, "Searched address is not part of the network\n");
--LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
-+struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
- if (--tree->refcount > 0)
- return tree;
+ loc_network_unref(*network);
+@@ -832,17 +845,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
-@@ -975,13 +975,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
- return 0;
- }
+ *network = NULL;
--LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
-+int loc_network_tree_dump(struct loc_network_tree* tree) {
- DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
++#ifdef ENABLE_DEBUG
+ // Save start time
+ clock_t start = clock();
++#endif
- return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
- }
+ int r = __loc_database_lookup(db, address, network, &network_address,
+ db->network_nodes_v1, 0);
--LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
-+int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
- DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
++#ifdef ENABLE_DEBUG
+ clock_t end = clock();
- struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
-@@ -1012,7 +1012,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
- return 0;
+ // Log how fast this has been
+ DEBUG(db->ctx, "Executed network search in %.4fms\n",
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
++#endif
+
+ return r;
}
+@@ -889,8 +906,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
+ off_t lo = 0;
+ off_t hi = db->countries_count - 1;
--LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
-+size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
- size_t counter = 0;
++#ifdef ENABLE_DEBUG
+ // Save start time
+ clock_t start = clock();
++#endif
+
+ while (lo <= hi) {
+ off_t i = (lo + hi) / 2;
+@@ -905,11 +924,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
+ int result = strcmp(code, cc);
+
+ if (result == 0) {
++#ifdef ENABLE_DEBUG
+ clock_t end = clock();
+
+ // Log how fast this has been
+ DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
++#endif
+
+ return 0;
+ }
+@@ -932,8 +953,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
+
+ // Enumerator
+
++static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
++ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
++
++ // Release all references
++ loc_database_unref(enumerator->db);
++ loc_unref(enumerator->ctx);
++
++ if (enumerator->string)
++ free(enumerator->string);
++
++ if (enumerator->countries)
++ loc_country_list_unref(enumerator->countries);
++
++ if (enumerator->asns)
++ loc_as_list_unref(enumerator->asns);
++
++ // Free network search
++ free(enumerator->networks_visited);
++
++ // Free subnet stack
++ if (enumerator->stack)
++ loc_network_list_unref(enumerator->stack);
++
++ free(enumerator);
++}
++
+ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
+- struct loc_database* db, enum loc_database_enumerator_mode mode) {
++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
+ struct loc_database_enumerator* e = calloc(1, sizeof(*e));
+ if (!e)
+ return -ENOMEM;
+@@ -944,11 +991,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
+ e->mode = mode;
+ e->refcount = 1;
+
++ // Flatten output?
++ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
++
+ // Initialise graph search
+- //e->network_stack[++e->network_stack_depth] = 0;
+ e->network_stack_depth = 1;
+ e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
- int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
-@@ -1034,11 +1034,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
- return counter;
- }
++ // Allocate stack
++ int r = loc_network_list_new(e->ctx, &e->stack);
++ if (r) {
++ loc_database_enumerator_free(e);
++ return r;
++ }
++
+ DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
--LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
-+size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
- return __loc_network_tree_count_nodes(tree->root);
+ *enumerator = e;
+@@ -961,22 +1017,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
+ return enumerator;
}
--LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
-+int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
- struct loc_network_tree_node* n = calloc(1, sizeof(*n));
- if (!n)
- return -ENOMEM;
-@@ -1053,7 +1053,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
+-static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
+- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
+-
+- // Release all references
+- loc_database_unref(enumerator->db);
+- loc_unref(enumerator->ctx);
+-
+- if (enumerator->string)
+- free(enumerator->string);
+-
+- // Free network search
+- free(enumerator->networks_visited);
+-
+- free(enumerator);
+-}
+-
+ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
+ if (!enumerator)
+ return NULL;
+@@ -998,40 +1038,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
return 0;
}
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
-+struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
- if (node)
- node->refcount++;
+-LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
+- // Set empty country code
+- if (!country_code || !*country_code) {
+- *enumerator->country_code = '\0';
+- return 0;
+- }
++LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
++ struct loc_database_enumerator* enumerator) {
++ if (!enumerator->countries)
++ return NULL;
-@@ -1076,7 +1076,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
- free(node);
- }
+- // Treat A1, A2, A3 as special country codes,
+- // but perform search for flags instead
+- if (strcmp(country_code, "A1") == 0) {
+- return loc_database_enumerator_set_flag(enumerator,
+- LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
+- } else if (strcmp(country_code, "A2") == 0) {
+- return loc_database_enumerator_set_flag(enumerator,
+- LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
+- } else if (strcmp(country_code, "A3") == 0) {
+- return loc_database_enumerator_set_flag(enumerator,
+- LOC_NETWORK_FLAG_ANYCAST);
+- }
++ return loc_country_list_ref(enumerator->countries);
++}
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
-+struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
- if (!node)
- return NULL;
+- // Country codes must be two characters
+- if (!loc_country_code_is_valid(country_code))
+- return -EINVAL;
++LOC_EXPORT int loc_database_enumerator_set_countries(
++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
++ if (enumerator->countries)
++ loc_country_list_unref(enumerator->countries);
-@@ -1087,7 +1087,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
- return NULL;
- }
+- for (unsigned int i = 0; i < 3; i++) {
+- enumerator->country_code[i] = country_code[i];
+- }
++ enumerator->countries = loc_country_list_ref(countries);
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
-+struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
- if (index == 0)
- node = node->zero;
- else
-@@ -1099,11 +1099,11 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
- return loc_network_tree_node_ref(node);
+ return 0;
}
--LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
-+int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
- return (!!node->network);
- }
+-LOC_EXPORT int loc_database_enumerator_set_asn(
+- struct loc_database_enumerator* enumerator, unsigned int asn) {
+- enumerator->asn = asn;
++LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
++ struct loc_database_enumerator* enumerator) {
++ if (!enumerator->asns)
++ return NULL;
++
++ return loc_as_list_ref(enumerator->asns);
++}
++
++LOC_EXPORT int loc_database_enumerator_set_asns(
++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
++ if (enumerator->asns)
++ loc_as_list_unref(enumerator->asns);
++
++ enumerator->asns = loc_as_list_ref(asns);
--LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
-+struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
- return loc_network_ref(node->network);
+ return 0;
+ }
+@@ -1110,16 +1148,64 @@ static int loc_database_enumerator_stack_push_node(
+ return 0;
}
-diff --git a/src/test-network.c b/src/test-network.c
-index af1b2e6..7c90224 100644
---- a/src/test-network.c
-+++ b/src/test-network.c
-@@ -37,12 +37,14 @@ int main(int argc, char** argv) {
- // Enable debug logging
- loc_set_log_priority(ctx, LOG_DEBUG);
-
-+#if 0
- struct loc_network_tree* tree;
- err = loc_network_tree_new(ctx, &tree);
- if (err) {
- fprintf(stderr, "Could not create the network tree\n");
- exit(EXIT_FAILURE);
- }
-+#endif
+-LOC_EXPORT int loc_database_enumerator_next_network(
+- struct loc_database_enumerator* enumerator, struct loc_network** network) {
+- // Reset network
+- *network = NULL;
++static int loc_database_enumerator_filter_network(
++ struct loc_database_enumerator* enumerator, struct loc_network* network) {
++ // Skip if the family does not match
++ if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
++ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
++ return 1;
++ }
- // Create a network
- struct loc_network* network1;
-@@ -58,12 +60,14 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
+- // Do not do anything if not in network mode
+- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
+- return 0;
++ // Skip if the country code does not match
++ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
++ const char* country_code = loc_network_get_country_code(network);
-+#if 0
- // Adding network to the tree
- err = loc_network_tree_add_network(tree, network1);
- if (err) {
- fprintf(stderr, "Could not add network to the tree\n");
- exit(EXIT_FAILURE);
- }
-+#endif
+- int r;
++ if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
++ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
++ return 1;
++ }
++ }
++
++ // Skip if the ASN does not match
++ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
++ uint32_t asn = loc_network_get_asn(network);
++
++ if (!loc_as_list_contains_number(enumerator->asns, asn)) {
++ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
++ return 1;
++ }
++ }
++
++ // Skip if flags do not match
++ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
++ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
++ return 1;
++ }
++
++ // Do not filter
++ return 0;
++}
++
++static int __loc_database_enumerator_next_network(
++ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
++ // Return top element from the stack
++ while (1) {
++ *network = loc_network_list_pop_first(enumerator->stack);
++
++ // Stack is empty
++ if (!*network)
++ break;
++
++ // Throw away any networks by filter
++ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
++ loc_network_unref(*network);
++ *network = NULL;
++ continue;
++ }
++
++ // Return result
++ return 0;
++ }
- // Check if the first and last addresses are correct
- char* string = loc_network_format_first_address(network1);
-@@ -101,6 +105,7 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
+ DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
+ enumerator->network_stack_depth);
+@@ -1149,7 +1235,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
+ enumerator->db->network_nodes_v1 + node->offset;
-+#if 0
- // Adding network to the tree
- err = loc_network_tree_add_network(tree, network2);
- if (err) {
-@@ -117,6 +122,7 @@ int main(int argc, char** argv) {
+ // Add edges to stack
+- r = loc_database_enumerator_stack_push_node(enumerator,
++ int r = loc_database_enumerator_stack_push_node(enumerator,
+ be32toh(n->one), 1, node->depth + 1);
- size_t nodes = loc_network_tree_count_nodes(tree);
- printf("The tree has %zu nodes\n", nodes);
-+#endif
+ if (r)
+@@ -1175,56 +1261,145 @@ LOC_EXPORT int loc_database_enumerator_next_network(
+ if (r)
+ return r;
- // Check equals function
- err = loc_network_eq(network1, network1);
-@@ -260,7 +266,10 @@ int main(int argc, char** argv) {
- loc_network_unref(network2);
- loc_network_unref(network3);
- loc_network_unref(network4);
-+
-+#if 0
- loc_network_tree_unref(tree);
-+#endif
+- // Check if we are interested in this network
++ // Return all networks when the filter is disabled
++ if (!filter)
++ return 0;
- // And open it again from disk
- struct loc_database* db;
---
-2.20.1
-
-From c242f7325bd6fc4ba26047ac24196d1c161c6e01 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 13 Nov 2020 12:09:03 +0000
-Subject: [PATCH 054/111] python: Move tree flattening into C
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/database.c | 8 ++-
- src/python/export.py | 138 +++++-------------------------------------
- 2 files changed, 21 insertions(+), 125 deletions(-)
-
-diff --git a/src/python/database.c b/src/python/database.c
-index 7f8c2c2..d169547 100644
---- a/src/python/database.c
-+++ b/src/python/database.c
-@@ -258,17 +258,19 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
- }
+- // Skip if the family does not match
+- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
++ // Check if we are interested in this network
++ if (loc_database_enumerator_filter_network(enumerator, *network)) {
+ loc_network_unref(*network);
+ *network = NULL;
- static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
-- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
-+ char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
- const char* country_code = NULL;
- unsigned int asn = 0;
- int flags = 0;
- int family = 0;
-+ int flatten = 0;
+ continue;
+ }
-- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
-+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
- return NULL;
+- // Skip if the country code does not match
+- if (*enumerator->country_code &&
+- !loc_network_match_country_code(*network, enumerator->country_code)) {
+- loc_network_unref(*network);
+- *network = NULL;
++ return 0;
++ }
++ }
- struct loc_database_enumerator* enumerator;
-- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
-+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
-+ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
- if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
- return NULL;
-diff --git a/src/python/export.py b/src/python/export.py
-index dd44332..be4a68e 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -87,58 +87,12 @@ class OutputWriter(object):
- def _write_network(self, network):
- self.f.write("%s\n" % network)
+- continue;
+- }
++ // Reached the end of the search
++ return 0;
++}
-- def write(self, network, subnets):
-+ def write(self, network):
- if self.flatten and self._flatten(network):
- log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
- return
+- // Skip if the ASN does not match
+- if (enumerator->asn &&
+- !loc_network_match_asn(*network, enumerator->asn)) {
+- loc_network_unref(*network);
+- *network = NULL;
++static int __loc_database_enumerator_next_network_flattened(
++ struct loc_database_enumerator* enumerator, struct loc_network** network) {
++ // Fetch the next network
++ int r = __loc_database_enumerator_next_network(enumerator, network, 1);
++ if (r)
++ return r;
-- # Convert network into a Python object
-- _network = ipaddress.ip_network("%s" % network)
--
-- # Write the network when it has no subnets
-- if not subnets:
-- log.debug("Writing %s to %s" % (_network, self.f))
-- return self._write_network(_network)
--
-- # Convert subnets into Python objects
-- _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
--
-- # Split the network into smaller bits so that
-- # we can accomodate for any gaps in it later
-- to_check = set()
-- for _subnet in _subnets:
-- to_check.update(
-- _network.address_exclude(_subnet)
-- )
--
-- # Clear the list of all subnets
-- subnets = []
--
-- # Check if all subnets to not overlap with anything else
-- while to_check:
-- subnet_to_check = to_check.pop()
--
-- for _subnet in _subnets:
-- # Drop this subnet if it equals one of the subnets
-- # or if it is subnet of one of them
-- if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
-- break
--
-- # Break it down if it overlaps
-- if subnet_to_check.overlaps(_subnet):
-- to_check.update(
-- subnet_to_check.address_exclude(_subnet)
-- )
-- break
--
-- # Add the subnet again as it passed the check
-- else:
-- subnets.append(subnet_to_check)
--
-- # Write all networks as compact as possible
-- for network in ipaddress.collapse_addresses(subnets):
-- log.debug("Writing %s to %s" % (network, self.f))
-- self._write_network(network)
-+ return self._write_network(network)
+- continue;
+- }
++ // End if we could not read another network
++ if (!*network)
++ return 0;
- def finish(self):
- """
-@@ -188,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter):
- mode = "wb"
+- // Skip if flags do not match
+- if (enumerator->flags &&
+- !loc_network_match_flag(*network, enumerator->flags)) {
+- loc_network_unref(*network);
+- *network = NULL;
++ struct loc_network* subnet = NULL;
++ struct loc_network_list* subnets;
- def _write_network(self, network):
-- for address in (network.network_address, network.broadcast_address):
-+ for address in (network.first_address, network.last_address):
- # Convert this into a string of bits
- bytes = socket.inet_pton(
- socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
-@@ -231,42 +185,21 @@ class Exporter(object):
- writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
+- continue;
++ // Create a list with all subnets
++ r = loc_network_list_new(enumerator->ctx, &subnets);
++ if (r)
++ return r;
++
++ // Search all subnets from the database
++ while (1) {
++ // Fetch the next network in line
++ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
++ if (r) {
++ loc_network_unref(subnet);
++ loc_network_list_unref(subnets);
++
++ return r;
++ }
++
++ // End if we did not receive another subnet
++ if (!subnet)
++ break;
++
++ // Collect all subnets in a list
++ if (loc_network_is_subnet(*network, subnet)) {
++ r = loc_network_list_push(subnets, subnet);
++ if (r) {
++ loc_network_unref(subnet);
++ loc_network_list_unref(subnets);
++
++ return r;
+ }
- # Get all networks that match the family
-- networks = self.db.search_networks(family=family)
--
-- # Create a stack with all networks in order where we can put items back
-- # again and retrieve them in the next iteration.
-- networks = BufferedStack(networks)
-+ networks = self.db.search_networks(family=family, flatten=True)
+- return 0;
++ loc_network_unref(subnet);
++ continue;
+ }
++
++ // If this is not a subnet, we push it back onto the stack and break
++ r = loc_network_list_push(enumerator->stack, subnet);
++ if (r) {
++ loc_network_unref(subnet);
++ loc_network_list_unref(subnets);
++
++ return r;
++ }
++
++ loc_network_unref(subnet);
++ break;
+ }
- # Walk through all networks
- for network in networks:
-- # Collect all networks which are a subnet of network
-- subnets = []
-- for subnet in networks:
-- # If the next subnet was not a subnet, we have to push
-- # it back on the stack and break this loop
-- if not subnet.is_subnet_of(network):
-- networks.push(subnet)
-- break
--
-- subnets.append(subnet)
--
- # Write matching countries
-- if network.country_code and network.country_code in writers:
-- # Mismatching subnets
-- gaps = [
-- subnet for subnet in subnets if not network.country_code == subnet.country_code
-- ]
--
-- writers[network.country_code].write(network, gaps)
-+ try:
-+ writers[network.country_code].write(network)
-+ except KeyError:
-+ pass
-
- # Write matching ASNs
-- if network.asn and network.asn in writers:
-- # Mismatching subnets
-- gaps = [
-- subnet for subnet in subnets if not network.asn == subnet.asn
-- ]
--
-- writers[network.asn].write(network, gaps)
-+ try:
-+ writers[network.asn].write(network)
-+ except KeyError:
-+ pass
+- // Reached the end of the search
++ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
++
++ // We can abort here if the network has no subnets
++ if (loc_network_list_empty(subnets)) {
++ loc_network_list_unref(subnets);
++
++ return 0;
++ }
- # Handle flags
- for flag in flags:
-@@ -274,19 +207,10 @@ class Exporter(object):
- # Fetch the "fake" country code
- country = flags[flag]
+- // Mark all nodes as non-visited
+- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
+- enumerator->networks_visited[i] = 0;
++ // If the network has any subnets, we will break it into smaller parts
++ // without the subnets.
++ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
++ if (!excluded) {
++ loc_network_list_unref(subnets);
++ return -1;
++ }
++
++ // Merge subnets onto the stack
++ r = loc_network_list_merge(enumerator->stack, subnets);
++ if (r) {
++ loc_network_list_unref(subnets);
++ loc_network_list_unref(excluded);
++
++ return r;
++ }
++
++ // Push excluded list onto the stack
++ r = loc_network_list_merge(enumerator->stack, excluded);
++ if (r) {
++ loc_network_list_unref(subnets);
++ loc_network_list_unref(excluded);
++
++ return r;
++ }
++
++ loc_network_list_unref(subnets);
++ loc_network_list_unref(excluded);
++
++ // Replace network with the first one from the stack
++ loc_network_unref(*network);
++ *network = loc_network_list_pop_first(enumerator->stack);
-- if not country in writers:
-- continue
--
-- gaps = [
-- subnet for subnet in subnets
-- if not subnet.has_flag(flag)
-- ]
--
-- writers[country].write(network, gaps)
--
-- # Push all subnets back onto the stack
-- for subnet in reversed(subnets):
-- networks.push(subnet)
-+ try:
-+ writers[country].write(network)
-+ except KeyError:
-+ pass
-
- # Write everything to the filesystem
- for writer in writers.values():
-@@ -298,33 +222,3 @@ class Exporter(object):
- )
+ return 0;
+ }
- return os.path.join(directory, filename)
--
--
--class BufferedStack(object):
-- """
-- This class takes an iterator and when being iterated
-- over it returns objects from that iterator for as long
-- as there are any.
--
-- It additionally has a function to put an item back on
-- the back so that it will be returned again at the next
-- iteration.
-- """
-- def __init__(self, iterator):
-- self.iterator = iterator
-- self.stack = []
--
-- def __iter__(self):
-- return self
--
-- def __next__(self):
-- if self.stack:
-- return self.stack.pop(0)
--
-- return next(self.iterator)
--
-- def push(self, elem):
-- """
-- Takes an element and puts it on the stack
-- """
-- self.stack.insert(0, elem)
---
-2.20.1
-
-From e0b9ff5f38beb0d560b16db881647e5a75127df1 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Sun, 15 Nov 2020 15:02:28 +0000
-Subject: [PATCH 055/111] Move network lists into an own file
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- Makefile.am | 2 +
- src/libloc.sym | 1 +
- src/loc/network-list.h | 37 +++++++
- src/loc/network.h | 20 +---
- src/network-list.c | 224 +++++++++++++++++++++++++++++++++++++++++
- src/network.c | 207 +------------------------------------
- src/python/network.c | 1 +
- 7 files changed, 269 insertions(+), 223 deletions(-)
- create mode 100644 src/loc/network-list.h
- create mode 100644 src/network-list.c
-
-diff --git a/Makefile.am b/Makefile.am
-index a0431a6..f0d8c4c 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -96,6 +96,7 @@ pkginclude_HEADERS = \
- src/loc/database.h \
- src/loc/format.h \
- src/loc/network.h \
-+ src/loc/network-list.h \
- src/loc/private.h \
- src/loc/stringpool.h \
- src/loc/resolv.h \
-@@ -110,6 +111,7 @@ src_libloc_la_SOURCES = \
- src/country.c \
- src/database.c \
- src/network.c \
-+ src/network-list.c \
- src/resolv.c \
- src/stringpool.c \
- src/writer.c
++LOC_EXPORT int loc_database_enumerator_next_network(
++ struct loc_database_enumerator* enumerator, struct loc_network** network) {
++ // Do not do anything if not in network mode
++ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
++ return 0;
++
++ // Flatten output?
++ if (enumerator->flatten)
++ return __loc_database_enumerator_next_network_flattened(enumerator, network);
++
++ return __loc_database_enumerator_next_network(enumerator, network, 1);
++}
++
+ LOC_EXPORT int loc_database_enumerator_next_country(
+ struct loc_database_enumerator* enumerator, struct loc_country** country) {
+ *country = NULL;
diff --git a/src/libloc.sym b/src/libloc.sym
-index 6139db6..453a1be 100644
+index b8296eb..ee333f1 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
-@@ -87,6 +87,7 @@ global:
+@@ -37,6 +37,18 @@ global:
+ loc_as_set_name;
+ loc_as_unref;
+
++ # AS List
++ loc_as_list_append;
++ loc_as_list_clear;
++ loc_as_list_contains;
++ loc_as_list_contains_number;
++ loc_as_list_empty;
++ loc_as_list_get;
++ loc_as_list_new;
++ loc_as_list_ref;
++ loc_as_list_size;
++ loc_as_list_unref;
++
+ # Country
+ loc_country_cmp;
+ loc_country_code_is_valid;
+@@ -49,6 +61,18 @@ global:
+ loc_country_set_name;
+ loc_country_unref;
+
++ # Country List
++ loc_country_list_append;
++ loc_country_list_clear;
++ loc_country_list_contains;
++ loc_country_list_contains_code;
++ loc_country_list_empty;
++ loc_country_list_get;
++ loc_country_list_new;
++ loc_country_list_ref;
++ loc_country_list_size;
++ loc_country_list_unref;
++
+ # Database
+ loc_database_add_as;
+ loc_database_count_as;
+@@ -66,13 +90,15 @@ global:
+ loc_database_verify;
+
+ # Database Enumerator
++ loc_database_enumerator_get_asns;
++ loc_database_enumerator_get_countries;
+ loc_database_enumerator_new;
+ loc_database_enumerator_next_as;
+ loc_database_enumerator_next_country;
+ loc_database_enumerator_next_network;
+ loc_database_enumerator_ref;
+- loc_database_enumerator_set_asn;
+- loc_database_enumerator_set_country_code;
++ loc_database_enumerator_set_asns;
++ loc_database_enumerator_set_countries;
+ loc_database_enumerator_set_family;
+ loc_database_enumerator_set_flag;
+ loc_database_enumerator_set_string;
+@@ -80,24 +106,48 @@ global:
+
+ # Network
+ loc_network_address_family;
++ loc_network_cmp;
++ loc_network_exclude;
++ loc_network_exclude_list;
+ loc_network_format_first_address;
loc_network_format_last_address;
loc_network_get_asn;
loc_network_get_country_code;
-+ loc_network_gt;
++ loc_network_get_first_address;
++ loc_network_get_last_address;
loc_network_has_flag;
- loc_network_is_subnet;
- loc_network_is_subnet_of;
-diff --git a/src/loc/network-list.h b/src/loc/network-list.h
+- loc_network_is_subnet_of;
++ loc_network_is_subnet;
++ loc_network_match_address;
+ loc_network_match_asn;
+ loc_network_match_country_code;
+ loc_network_match_flag;
+ loc_network_new;
+ loc_network_new_from_string;
++ loc_network_overlaps;
++ loc_network_prefix;
+ loc_network_ref;
+ loc_network_set_asn;
+ loc_network_set_country_code;
+ loc_network_set_flag;
+ loc_network_str;
++ loc_network_subnets;
+ loc_network_unref;
+
++ # Network List
++ loc_network_list_clear;
++ loc_network_list_contains;
++ loc_network_list_dump;
++ loc_network_list_empty;
++ loc_network_list_get;
++ loc_network_list_merge;
++ loc_network_list_new;
++ loc_network_list_pop;
++ loc_network_list_pop_first;
++ loc_network_list_push;
++ loc_network_list_ref;
++ loc_network_list_size;
++ loc_network_list_unref;
++
+ # Writer
+ loc_writer_add_as;
+ loc_writer_add_country;
+diff --git a/src/loc/as-list.h b/src/loc/as-list.h
new file mode 100644
-index 0000000..af3b28d
+index 0000000..7b5c4e8
--- /dev/null
-+++ b/src/loc/network-list.h
-@@ -0,0 +1,37 @@
++++ b/src/loc/as-list.h
+@@ -0,0 +1,41 @@
+/*
+ libloc - A library to determine the location of someone on the Internet
+
-+ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
++ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ Lesser General Public License for more details.
+*/
+
-+#ifndef LIBLOC_NETWORK_LIST_H
-+#define LIBLOC_NETWORK_LIST_H
++#ifndef LIBLOC_AS_LIST_H
++#define LIBLOC_AS_LIST_H
+
-+struct loc_network_list;
-+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
-+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
-+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
-+size_t loc_network_list_size(struct loc_network_list* list);
-+int loc_network_list_empty(struct loc_network_list* list);
-+void loc_network_list_clear(struct loc_network_list* list);
-+void loc_network_list_dump(struct loc_network_list* list);
-+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
-+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
-+struct loc_network* loc_network_list_pop(struct loc_network_list* list);
-+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
-+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
-+void loc_network_list_sort(struct loc_network_list* list);
-+void loc_network_list_reverse(struct loc_network_list* list);
-+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
++#include <loc/as.h>
++#include <loc/libloc.h>
++
++struct loc_as_list;
++
++int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
++struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
++struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
++
++size_t loc_as_list_size(struct loc_as_list* list);
++int loc_as_list_empty(struct loc_as_list* list);
++void loc_as_list_clear(struct loc_as_list* list);
++
++struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
++int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
++
++int loc_as_list_contains(
++ struct loc_as_list* list, struct loc_as* as);
++int loc_as_list_contains_number(
++ struct loc_as_list* list, uint32_t number);
+
+#endif
-diff --git a/src/loc/network.h b/src/loc/network.h
-index 203e61c..d86b685 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -21,6 +21,7 @@
-
- #include <loc/libloc.h>
- #include <loc/format.h>
-+#include <loc/network-list.h>
-
- enum loc_network_flags {
- LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
-@@ -55,6 +56,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
- int loc_network_match_flag(struct loc_network* network, uint32_t flag);
-
- int loc_network_eq(struct loc_network* self, struct loc_network* other);
-+int loc_network_gt(struct loc_network* self, struct loc_network* other);
- int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
- int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
- int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
-@@ -64,24 +66,6 @@ struct loc_network_list* loc_network_exclude(
- struct loc_network_list* loc_network_exclude_list(
- struct loc_network* network, struct loc_network_list* list);
-
--// List
--struct loc_network_list;
--int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
--struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
--struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
--size_t loc_network_list_size(struct loc_network_list* list);
--int loc_network_list_empty(struct loc_network_list* list);
--void loc_network_list_clear(struct loc_network_list* list);
--void loc_network_list_dump(struct loc_network_list* list);
--struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
--int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
--struct loc_network* loc_network_list_pop(struct loc_network_list* list);
--struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
--int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
--void loc_network_list_sort(struct loc_network_list* list);
--void loc_network_list_reverse(struct loc_network_list* list);
--int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
--
- #ifdef LIBLOC_PRIVATE
-
- int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
-diff --git a/src/network-list.c b/src/network-list.c
+diff --git a/src/loc/country-list.h b/src/loc/country-list.h
new file mode 100644
-index 0000000..1f6e80e
+index 0000000..a7f818a
--- /dev/null
-+++ b/src/network-list.c
-@@ -0,0 +1,224 @@
++++ b/src/loc/country-list.h
+@@ -0,0 +1,43 @@
+/*
+ libloc - A library to determine the location of someone on the Internet
+
-+ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
++ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ Lesser General Public License for more details.
-+*/
-+
-+#include <errno.h>
-+#include <stdlib.h>
-+
-+#include <loc/libloc.h>
-+#include <loc/network.h>
-+#include <loc/private.h>
-+
-+struct loc_network_list {
-+ struct loc_ctx* ctx;
-+ int refcount;
-+
-+ struct loc_network* list[1024];
-+ size_t size;
-+ size_t max_size;
-+};
-+
-+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
-+ struct loc_network_list** list) {
-+ struct loc_network_list* l = calloc(1, sizeof(*l));
-+ if (!l)
-+ return -ENOMEM;
-+
-+ l->ctx = loc_ref(ctx);
-+ l->refcount = 1;
-+
-+ // Do not allow this list to grow larger than this
-+ l->max_size = 1024;
-+
-+ DEBUG(l->ctx, "Network list allocated at %p\n", l);
-+ *list = l;
-+ return 0;
-+}
-+
-+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
-+ list->refcount++;
-+
-+ return list;
-+}
-+
-+static void loc_network_list_free(struct loc_network_list* list) {
-+ DEBUG(list->ctx, "Releasing network list at %p\n", list);
-+
-+ for (unsigned int i = 0; i < list->size; i++)
-+ loc_network_unref(list->list[i]);
-+
-+ loc_unref(list->ctx);
-+ free(list);
-+}
-+
-+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
-+ if (!list)
-+ return NULL;
-+
-+ if (--list->refcount > 0)
-+ return list;
-+
-+ loc_network_list_free(list);
-+ return NULL;
-+}
-+
-+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
-+ return list->size;
-+}
-+
-+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
-+ return list->size == 0;
-+}
-+
-+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
-+ for (unsigned int i = 0; i < list->size; i++)
-+ loc_network_unref(list->list[i]);
-+
-+ list->size = 0;
-+}
-+
-+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
-+ struct loc_network* network;
-+ char* s;
-+
-+ for (unsigned int i = 0; i < list->size; i++) {
-+ network = list->list[i];
-+
-+ s = loc_network_str(network);
-+
-+ INFO(list->ctx, "%s\n", s);
-+ free(s);
-+ }
-+}
-+
-+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
-+ // Check index
-+ if (index >= list->size)
-+ return NULL;
-+
-+ return loc_network_ref(list->list[index]);
-+}
-+
-+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
-+ // Do not add networks that are already on the list
-+ if (loc_network_list_contains(list, network))
-+ return 0;
-+
-+ // Check if we have space left
-+ if (list->size == list->max_size) {
-+ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
-+ return -ENOMEM;
-+ }
-+
-+ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
-+
-+ list->list[list->size++] = loc_network_ref(network);
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
-+ // Return nothing when empty
-+ if (loc_network_list_empty(list)) {
-+ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
-+ return NULL;
-+ }
-+
-+ struct loc_network* network = list->list[--list->size];
-+
-+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
-+
-+ return network;
-+}
-+
-+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
-+ // Return nothing when empty
-+ if (loc_network_list_empty(list)) {
-+ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
-+ return NULL;
-+ }
-+
-+ struct loc_network* network = list->list[0];
++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ Lesser General Public License for more details.
++*/
+
-+ // Move all elements to the top of the stack
-+ for (unsigned int i = 0; i < --list->size; i++) {
-+ list->list[i] = list->list[i+1];
-+ }
++#ifndef LIBLOC_COUNTRY_LIST_H
++#define LIBLOC_COUNTRY_LIST_H
+
-+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
++#include <stdlib.h>
+
-+ return network;
-+}
++#include <loc/libloc.h>
++#include <loc/country.h>
+
-+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
-+ for (unsigned int i = 0; i < list->size; i++) {
-+ if (loc_network_eq(list->list[i], network))
-+ return 1;
-+ }
++struct loc_country_list;
+
-+ return 0;
-+}
++int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
++struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
++struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
+
-+static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
-+ // Do nothing for invalid indices
-+ if (i1 >= list->size || i2 >= list->size)
-+ return;
++size_t loc_country_list_size(struct loc_country_list* list);
++int loc_country_list_empty(struct loc_country_list* list);
++void loc_country_list_clear(struct loc_country_list* list);
+
-+ struct loc_network* network1 = list->list[i1];
-+ struct loc_network* network2 = list->list[i2];
++struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
++int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
+
-+ list->list[i1] = network2;
-+ list->list[i2] = network1;
-+}
++int loc_country_list_contains(
++ struct loc_country_list* list, struct loc_country* country);
++int loc_country_list_contains_code(
++ struct loc_country_list* list, const char* code);
+
-+LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
-+ unsigned int i = 0;
-+ unsigned int j = list->size - 1;
++#endif
+diff --git a/src/loc/database.h b/src/loc/database.h
+index 43173dd..70801f0 100644
+--- a/src/loc/database.h
++++ b/src/loc/database.h
+@@ -25,6 +25,7 @@
+ #include <loc/network.h>
+ #include <loc/as.h>
+ #include <loc/country.h>
++#include <loc/country-list.h>
+
+ struct loc_database;
+ int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
+@@ -55,15 +56,24 @@ enum loc_database_enumerator_mode {
+ LOC_DB_ENUMERATE_COUNTRIES = 3,
+ };
+
++enum loc_database_enumerator_flags {
++ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
++};
+
-+ while (i < j) {
-+ loc_network_list_swap(list, i++, j--);
-+ }
-+}
+ struct loc_database_enumerator;
+ int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
+- struct loc_database* db, enum loc_database_enumerator_mode mode);
++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
+ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
+ struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
+
+ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
+-int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
+-int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
++struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
++int loc_database_enumerator_set_countries(
++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
++struct loc_as_list* loc_database_enumerator_get_asns(
++ struct loc_database_enumerator* enumerator);
++int loc_database_enumerator_set_asns(
++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
+ int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
+ int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
+ int loc_database_enumerator_next_as(
+diff --git a/src/loc/network-list.h b/src/loc/network-list.h
+new file mode 100644
+index 0000000..bee21c4
+--- /dev/null
++++ b/src/loc/network-list.h
+@@ -0,0 +1,37 @@
++/*
++ libloc - A library to determine the location of someone on the Internet
+
-+LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
-+ unsigned int n = list->size;
-+ int swapped;
++ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
+
-+ do {
-+ swapped = 0;
++ This library is free software; you can redistribute it and/or
++ modify it under the terms of the GNU Lesser General Public
++ License as published by the Free Software Foundation; either
++ version 2.1 of the License, or (at your option) any later version.
+
-+ for (unsigned int i = 1; i < n; i++) {
-+ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
-+ loc_network_list_swap(list, i-1, i);
-+ swapped = 1;
-+ }
-+ }
++ This library is distributed in the hope that it will be useful,
++ but WITHOUT ANY WARRANTY; without even the implied warranty of
++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ Lesser General Public License for more details.
++*/
+
-+ n--;
-+ } while (swapped);
-+}
++#ifndef LIBLOC_NETWORK_LIST_H
++#define LIBLOC_NETWORK_LIST_H
+
-+LOC_EXPORT int loc_network_list_merge(
-+ struct loc_network_list* self, struct loc_network_list* other) {
-+ int r;
++#include <loc/network.h>
+
-+ for (unsigned int i = 0; i < other->size; i++) {
-+ r = loc_network_list_push(self, other->list[i]);
-+ if (r)
-+ return r;
-+ }
++struct loc_network_list;
++int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
++struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
++struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
++size_t loc_network_list_size(struct loc_network_list* list);
++int loc_network_list_empty(struct loc_network_list* list);
++void loc_network_list_clear(struct loc_network_list* list);
++void loc_network_list_dump(struct loc_network_list* list);
++struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
++int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
++struct loc_network* loc_network_list_pop(struct loc_network_list* list);
++struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
++int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
++int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
+
-+ return 0;
-+}
-diff --git a/src/network.c b/src/network.c
-index d015579..28ca2df 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -29,6 +29,7 @@
- #include <loc/compat.h>
- #include <loc/country.h>
- #include <loc/network.h>
++#endif
+diff --git a/src/loc/network.h b/src/loc/network.h
+index 70c3803..af3dafd 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -21,6 +21,7 @@
+
+ #include <loc/libloc.h>
+ #include <loc/format.h>
+#include <loc/network-list.h>
- #include <loc/private.h>
- struct loc_network {
-@@ -436,7 +437,7 @@ LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* othe
- return 1;
- }
+ enum loc_network_flags {
+ LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
+@@ -37,8 +38,11 @@ struct loc_network* loc_network_ref(struct loc_network* network);
+ struct loc_network* loc_network_unref(struct loc_network* network);
+ char* loc_network_str(struct loc_network* network);
+ int loc_network_address_family(struct loc_network* network);
++unsigned int loc_network_prefix(struct loc_network* network);
--static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
-+LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
- // Families must match
- if (self->family != other->family)
- return -1;
-@@ -1106,207 +1107,3 @@ int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
- struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
- return loc_network_ref(node->network);
- }
--
--// List
--
--struct loc_network_list {
-- struct loc_ctx* ctx;
-- int refcount;
--
-- struct loc_network* list[1024];
-- size_t size;
-- size_t max_size;
--};
--
--LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
-- struct loc_network_list** list) {
-- struct loc_network_list* l = calloc(1, sizeof(*l));
-- if (!l)
-- return -ENOMEM;
--
-- l->ctx = loc_ref(ctx);
-- l->refcount = 1;
--
-- // Do not allow this list to grow larger than this
-- l->max_size = 1024;
--
-- DEBUG(l->ctx, "Network list allocated at %p\n", l);
-- *list = l;
-- return 0;
--}
--
--LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
-- list->refcount++;
--
-- return list;
--}
--
--static void loc_network_list_free(struct loc_network_list* list) {
-- DEBUG(list->ctx, "Releasing network list at %p\n", list);
--
-- for (unsigned int i = 0; i < list->size; i++)
-- loc_network_unref(list->list[i]);
--
-- loc_unref(list->ctx);
-- free(list);
--}
--
--LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
-- if (!list)
-- return NULL;
--
-- if (--list->refcount > 0)
-- return list;
--
-- loc_network_list_free(list);
-- return NULL;
--}
--
--LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
-- return list->size;
--}
--
--LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
-- return list->size == 0;
--}
--
--LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
-- for (unsigned int i = 0; i < list->size; i++)
-- loc_network_unref(list->list[i]);
--
-- list->size = 0;
--}
--
--LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
-- struct loc_network* network;
-- char* s;
--
-- for (unsigned int i = 0; i < list->size; i++) {
-- network = list->list[i];
--
-- s = loc_network_str(network);
--
-- INFO(list->ctx, "%s\n", s);
-- free(s);
-- }
--}
--
--LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
-- // Check index
-- if (index >= list->size)
-- return NULL;
--
-- return loc_network_ref(list->list[index]);
--}
--
--LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
-- // Do not add networks that are already on the list
-- if (loc_network_list_contains(list, network))
-- return 0;
--
-- // Check if we have space left
-- if (list->size == list->max_size) {
-- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
-- return -ENOMEM;
-- }
--
-- DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
--
-- list->list[list->size++] = loc_network_ref(network);
--
-- return 0;
--}
--
--LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
-- // Return nothing when empty
-- if (loc_network_list_empty(list)) {
-- DEBUG(list->ctx, "%p: Popped empty stack\n", list);
-- return NULL;
-- }
--
-- struct loc_network* network = list->list[--list->size];
--
-- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
--
-- return network;
--}
--
--LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
-- // Return nothing when empty
-- if (loc_network_list_empty(list)) {
-- DEBUG(list->ctx, "%p: Popped empty stack\n", list);
-- return NULL;
-- }
--
-- struct loc_network* network = list->list[0];
--
-- // Move all elements to the top of the stack
-- for (unsigned int i = 0; i < --list->size; i++) {
-- list->list[i] = list->list[i+1];
-- }
--
-- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
--
-- return network;
--}
--
--LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
-- for (unsigned int i = 0; i < list->size; i++) {
-- if (loc_network_eq(list->list[i], network))
-- return 1;
-- }
--
-- return 0;
--}
--
--static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
-- // Do nothing for invalid indices
-- if (i1 >= list->size || i2 >= list->size)
-- return;
--
-- struct loc_network* network1 = list->list[i1];
-- struct loc_network* network2 = list->list[i2];
--
-- list->list[i1] = network2;
-- list->list[i2] = network1;
--}
--
--LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
-- unsigned int i = 0;
-- unsigned int j = list->size - 1;
--
-- while (i < j) {
-- loc_network_list_swap(list, i++, j--);
-- }
--}
--
--LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
-- unsigned int n = list->size;
-- int swapped;
--
-- do {
-- swapped = 0;
--
-- for (unsigned int i = 1; i < n; i++) {
-- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
-- loc_network_list_swap(list, i-1, i);
-- swapped = 1;
-- }
-- }
--
-- n--;
-- } while (swapped);
--}
--
--LOC_EXPORT int loc_network_list_merge(
-- struct loc_network_list* self, struct loc_network_list* other) {
-- int r;
--
-- for (unsigned int i = 0; i < other->size; i++) {
-- r = loc_network_list_push(self, other->list[i]);
-- if (r)
-- return r;
-- }
--
-- return 0;
--}
-diff --git a/src/python/network.c b/src/python/network.c
-index 11f672b..ed91d65 100644
---- a/src/python/network.c
-+++ b/src/python/network.c
-@@ -20,6 +20,7 @@
++const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
+ char* loc_network_format_first_address(struct loc_network* network);
++const struct in6_addr* loc_network_get_last_address(struct loc_network* network);
+ char* loc_network_format_last_address(struct loc_network* network);
+ int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
- #include <loc/libloc.h>
- #include <loc/network.h>
-+#include <loc/network-list.h>
+@@ -54,7 +58,14 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
+ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
+ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
+
+-int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
++int loc_network_cmp(struct loc_network* self, struct loc_network* other);
++int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
++int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
++int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
++struct loc_network_list* loc_network_exclude(
++ struct loc_network* self, struct loc_network* other);
++struct loc_network_list* loc_network_exclude_list(
++ struct loc_network* network, struct loc_network_list* list);
+
+ #ifdef LIBLOC_PRIVATE
- #include "locationmodule.h"
- #include "network.h"
---
-2.20.1
-
-From e646a8f35ec7eff009414b3fd107c9af5cf39a86 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Mon, 16 Nov 2020 15:13:28 +0000
-Subject: [PATCH 056/111] Implement filtering for multiple countries in the
- enumerator
-
-This will allow us to speed up the export of the database
-if only a few countries should be returned.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- Makefile.am | 2 +
- src/country-list.c | 138 +++++++++++++++++++++++++++++++++++++++++
- src/country.c | 3 +
- src/database.c | 47 ++++++--------
- src/libloc.sym | 15 ++++-
- src/loc/country-list.h | 43 +++++++++++++
- src/loc/database.h | 5 +-
- src/python/database.c | 57 ++++++++++++++---
- 8 files changed, 274 insertions(+), 36 deletions(-)
- create mode 100644 src/country-list.c
- create mode 100644 src/loc/country-list.h
-
-diff --git a/Makefile.am b/Makefile.am
-index f0d8c4c..f4ca3c8 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -93,6 +93,7 @@ pkginclude_HEADERS = \
- src/loc/as.h \
- src/loc/compat.h \
- src/loc/country.h \
-+ src/loc/country-list.h \
- src/loc/database.h \
- src/loc/format.h \
- src/loc/network.h \
-@@ -109,6 +110,7 @@ src_libloc_la_SOURCES = \
- src/libloc.c \
- src/as.c \
- src/country.c \
-+ src/country-list.c \
- src/database.c \
- src/network.c \
- src/network-list.c \
-diff --git a/src/country-list.c b/src/country-list.c
+diff --git a/src/network-list.c b/src/network-list.c
new file mode 100644
-index 0000000..ae0d71a
+index 0000000..e2b20f3
--- /dev/null
-+++ b/src/country-list.c
-@@ -0,0 +1,138 @@
++++ b/src/network-list.c
+@@ -0,0 +1,295 @@
+/*
+ libloc - A library to determine the location of someone on the Internet
+
+
+#include <errno.h>
+#include <stdlib.h>
++#include <time.h>
+
-+#include <loc/country.h>
-+#include <loc/country-list.h>
++#include <loc/libloc.h>
++#include <loc/network.h>
+#include <loc/private.h>
+
-+struct loc_country_list {
++struct loc_network_list {
+ struct loc_ctx* ctx;
+ int refcount;
+
-+ struct loc_country* list[1024];
++ struct loc_network** elements;
++ size_t elements_size;
++
+ size_t size;
-+ size_t max_size;
+};
+
-+LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
-+ struct loc_country_list** list) {
-+ struct loc_country_list* l = calloc(1, sizeof(*l));
++static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
++ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
++ list, size, list->elements_size + size);
++
++ struct loc_network** elements = reallocarray(list->elements,
++ list->elements_size + size, sizeof(*list->elements));
++ if (!elements)
++ return -errno;
++
++ list->elements = elements;
++ list->elements_size += size;
++
++ return 0;
++}
++
++LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
++ struct loc_network_list** list) {
++ struct loc_network_list* l = calloc(1, sizeof(*l));
+ if (!l)
+ return -ENOMEM;
+
+ l->ctx = loc_ref(ctx);
+ l->refcount = 1;
+
-+ // Do not allow this list to grow larger than this
-+ l->max_size = 1024;
-+
-+ DEBUG(l->ctx, "Country list allocated at %p\n", l);
++ DEBUG(l->ctx, "Network list allocated at %p\n", l);
+ *list = l;
-+
+ return 0;
+}
+
-+LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
++LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
+ list->refcount++;
+
+ return list;
+}
+
-+static void loc_country_list_free(struct loc_country_list* list) {
-+ DEBUG(list->ctx, "Releasing country list at %p\n", list);
++static void loc_network_list_free(struct loc_network_list* list) {
++ DEBUG(list->ctx, "Releasing network list at %p\n", list);
+
-+ loc_country_list_clear(list);
++ // Remove all content
++ loc_network_list_clear(list);
+
+ loc_unref(list->ctx);
+ free(list);
+}
+
-+LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
++LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
+ if (!list)
+ return NULL;
+
+ if (--list->refcount > 0)
+ return list;
+
-+ loc_country_list_free(list);
++ loc_network_list_free(list);
+ return NULL;
+}
+
-+LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
++LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
+ return list->size;
+}
+
-+LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
++LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
+ return list->size == 0;
+}
+
-+LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
-+ for (unsigned int i = 0; i < list->size; i++)
-+ loc_country_unref(list->list[i]);
-+}
-+
-+LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
-+ // Check index
-+ if (index >= list->size)
-+ return NULL;
-+
-+ return loc_country_ref(list->list[index]);
-+}
-+
-+LOC_EXPORT int loc_country_list_append(
-+ struct loc_country_list* list, struct loc_country* country) {
-+ if (loc_country_list_contains(list, country))
-+ return 0;
-+
-+ // Check if we have space left
-+ if (list->size == list->max_size) {
-+ ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
-+ return -ENOMEM;
-+ }
-+
-+ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
-+
-+ list->list[list->size++] = loc_country_ref(country);
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT int loc_country_list_contains(
-+ struct loc_country_list* list, struct loc_country* country) {
-+ for (unsigned int i = 0; i < list->size; i++) {
-+ if (loc_country_cmp(country, list->list[i]) == 0)
-+ return 1;
-+ }
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT int loc_country_list_contains_code(
-+ struct loc_country_list* list, const char* code) {
-+ struct loc_country* country;
-+
-+ int r = loc_country_new(list->ctx, &country, code);
-+ if (r)
-+ return -1;
-+
-+ r = loc_country_list_contains(list, country);
-+ loc_country_unref(country);
-+
-+ return r;
-+}
-diff --git a/src/country.c b/src/country.c
-index 2ba93e6..7aac0db 100644
---- a/src/country.c
-+++ b/src/country.c
-@@ -34,6 +34,9 @@ struct loc_country {
- };
-
- LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
-+ if (!loc_country_code_is_valid(country_code))
-+ return -EINVAL;
-+
- struct loc_country* c = calloc(1, sizeof(*c));
- if (!c)
- return -ENOMEM;
-diff --git a/src/database.c b/src/database.c
-index b9d870f..29823b2 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -40,6 +40,7 @@
- #include <loc/as.h>
- #include <loc/compat.h>
- #include <loc/country.h>
-+#include <loc/country-list.h>
- #include <loc/database.h>
- #include <loc/format.h>
- #include <loc/network.h>
-@@ -99,7 +100,7 @@ struct loc_database_enumerator {
-
- // Search string
- char* string;
-- char country_code[3];
-+ struct loc_country_list* countries;
- uint32_t asn;
- enum loc_network_flags flags;
- int family;
-@@ -1017,33 +1018,20 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
- return 0;
- }
-
--LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
-- // Set empty country code
-- if (!country_code || !*country_code) {
-- *enumerator->country_code = '\0';
-- return 0;
-- }
-+LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
-+ struct loc_database_enumerator* enumerator) {
-+ if (!enumerator->countries)
-+ return NULL;
-
-- // Treat A1, A2, A3 as special country codes,
-- // but perform search for flags instead
-- if (strcmp(country_code, "A1") == 0) {
-- return loc_database_enumerator_set_flag(enumerator,
-- LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
-- } else if (strcmp(country_code, "A2") == 0) {
-- return loc_database_enumerator_set_flag(enumerator,
-- LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
-- } else if (strcmp(country_code, "A3") == 0) {
-- return loc_database_enumerator_set_flag(enumerator,
-- LOC_NETWORK_FLAG_ANYCAST);
-- }
-+ return loc_country_list_ref(enumerator->countries);
-+}
-
-- // Country codes must be two characters
-- if (!loc_country_code_is_valid(country_code))
-- return -EINVAL;
-+LOC_EXPORT int loc_database_enumerator_set_countries(
-+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
-+ if (enumerator->countries)
-+ loc_country_list_unref(enumerator->countries);
-
-- for (unsigned int i = 0; i < 3; i++) {
-- enumerator->country_code[i] = country_code[i];
-- }
-+ enumerator->countries = loc_country_list_ref(countries);
-
- return 0;
- }
-@@ -1129,6 +1117,12 @@ static int loc_database_enumerator_stack_push_node(
- return 0;
- }
-
-+static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
-+ const char* country_code = loc_network_get_country_code(network);
-+
-+ return loc_country_list_contains_code(countries, country_code);
-+}
++LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
++ if (!list->elements)
++ return;
+
- static int loc_database_enumerator_filter_network(
- struct loc_database_enumerator* enumerator, struct loc_network* network) {
- // Skip if the family does not match
-@@ -1136,8 +1130,7 @@ static int loc_database_enumerator_filter_network(
- return 1;
-
- // Skip if the country code does not match
-- if (*enumerator->country_code &&
-- !loc_network_match_country_code(network, enumerator->country_code))
-+ if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
- return 1;
-
- // Skip if the ASN does not match
-diff --git a/src/libloc.sym b/src/libloc.sym
-index 453a1be..40e9f88 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -49,6 +49,18 @@ global:
- loc_country_set_name;
- loc_country_unref;
-
-+ # Country List
-+ loc_country_list_append;
-+ loc_country_list_clear;
-+ loc_country_list_contains;
-+ loc_country_list_contains_code;
-+ loc_country_list_empty;
-+ loc_country_list_get;
-+ loc_country_list_new;
-+ loc_country_list_ref;
-+ loc_country_list_size;
-+ loc_country_list_unref;
++ for (unsigned int i = 0; i < list->size; i++)
++ loc_network_unref(list->elements[i]);
+
- # Database
- loc_database_add_as;
- loc_database_count_as;
-@@ -66,13 +78,14 @@ global:
- loc_database_verify;
-
- # Database Enumerator
-+ loc_database_enumerator_get_countries;
- loc_database_enumerator_new;
- loc_database_enumerator_next_as;
- loc_database_enumerator_next_country;
- loc_database_enumerator_next_network;
- loc_database_enumerator_ref;
- loc_database_enumerator_set_asn;
-- loc_database_enumerator_set_country_code;
-+ loc_database_enumerator_set_countries;
- loc_database_enumerator_set_family;
- loc_database_enumerator_set_flag;
- loc_database_enumerator_set_string;
-diff --git a/src/loc/country-list.h b/src/loc/country-list.h
-new file mode 100644
-index 0000000..a7f818a
---- /dev/null
-+++ b/src/loc/country-list.h
-@@ -0,0 +1,43 @@
-+/*
-+ libloc - A library to determine the location of someone on the Internet
++ free(list->elements);
++ list->elements = NULL;
++ list->elements_size = 0;
+
-+ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
++ list->size = 0;
++}
+
-+ This library is free software; you can redistribute it and/or
-+ modify it under the terms of the GNU Lesser General Public
-+ License as published by the Free Software Foundation; either
-+ version 2.1 of the License, or (at your option) any later version.
++LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
++ struct loc_network* network;
++ char* s;
+
-+ This library is distributed in the hope that it will be useful,
-+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ Lesser General Public License for more details.
-+*/
++ for (unsigned int i = 0; i < list->size; i++) {
++ network = list->elements[i];
+
-+#ifndef LIBLOC_COUNTRY_LIST_H
-+#define LIBLOC_COUNTRY_LIST_H
++ s = loc_network_str(network);
+
-+#include <stdlib.h>
++ INFO(list->ctx, "%4d: %s\n", i, s);
++ free(s);
++ }
++}
+
-+#include <loc/libloc.h>
-+#include <loc/country.h>
++LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
++ // Check index
++ if (index >= list->size)
++ return NULL;
+
-+struct loc_country_list;
++ return loc_network_ref(list->elements[index]);
++}
+
-+int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
-+struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
-+struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
++static off_t loc_network_list_find(struct loc_network_list* list,
++ struct loc_network* network, int* found) {
++ off_t lo = 0;
++ off_t hi = list->size - 1;
++ int result;
+
-+size_t loc_country_list_size(struct loc_country_list* list);
-+int loc_country_list_empty(struct loc_country_list* list);
-+void loc_country_list_clear(struct loc_country_list* list);
++ // Since we are working on an ordered list, there is often a good chance that
++ // the network we are looking for is at the end or has to go to the end.
++ if (hi >= 0) {
++ result = loc_network_cmp(network, list->elements[hi]);
+
-+struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
-+int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
++ // Match, so we are done
++ if (result == 0) {
++ *found = 1;
+
-+int loc_country_list_contains(
-+ struct loc_country_list* list, struct loc_country* country);
-+int loc_country_list_contains_code(
-+ struct loc_country_list* list, const char* code);
++ return hi;
++
++ // This needs to be added after the last one
++ } else if (result > 0) {
++ *found = 0;
++
++ return hi + 1;
++ }
++ }
+
++#ifdef ENABLE_DEBUG
++ // Save start time
++ clock_t start = clock();
+#endif
-diff --git a/src/loc/database.h b/src/loc/database.h
-index 14eb5ea..246e5c5 100644
---- a/src/loc/database.h
-+++ b/src/loc/database.h
-@@ -25,6 +25,7 @@
- #include <loc/network.h>
- #include <loc/as.h>
- #include <loc/country.h>
-+#include <loc/country-list.h>
-
- struct loc_database;
- int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
-@@ -66,7 +67,9 @@ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_
- struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
-
- int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
--int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
-+struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
-+int loc_database_enumerator_set_countries(
-+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
- int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
- int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
- int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
-diff --git a/src/python/database.c b/src/python/database.c
-index d169547..e6f6f37 100644
---- a/src/python/database.c
-+++ b/src/python/database.c
-@@ -258,14 +258,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
- }
-
- static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
-- char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
-- const char* country_code = NULL;
-+ char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
-+ PyObject* country_codes = NULL;
- unsigned int asn = 0;
- int flags = 0;
- int family = 0;
- int flatten = 0;
-
-- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
-+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
-+ &PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
- return NULL;
-
- struct loc_database_enumerator* enumerator;
-@@ -277,13 +278,55 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
- }
-
- // Set country code we are searching for
-- if (country_code) {
-- r = loc_database_enumerator_set_country_code(enumerator, country_code);
--
-+ if (country_codes) {
-+ struct loc_country_list* countries;
-+ r = loc_country_list_new(loc_ctx, &countries);
- if (r) {
-- PyErr_SetFromErrno(PyExc_SystemError);
-+ PyErr_SetString(PyExc_SystemError, "Could not create country list");
- return NULL;
- }
+
-+ for (unsigned int i = 0; i < PyList_Size(country_codes); i++) {
-+ PyObject* item = PyList_GetItem(country_codes, i);
++ off_t i = 0;
+
-+ if (!PyUnicode_Check(item)) {
-+ PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
-+ loc_country_list_unref(countries);
-+ return NULL;
-+ }
++ while (lo <= hi) {
++ i = (lo + hi) / 2;
+
-+ const char* country_code = PyUnicode_AsUTF8(item);
++ // Check if this is a match
++ result = loc_network_cmp(network, list->elements[i]);
+
-+ struct loc_country* country;
-+ r = loc_country_new(loc_ctx, &country, country_code);
-+ if (r) {
-+ if (r == -EINVAL) {
-+ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
-+ } else {
-+ PyErr_SetString(PyExc_SystemError, "Could not create country");
-+ }
++ if (result == 0) {
++ *found = 1;
+
-+ loc_country_list_unref(countries);
-+ return NULL;
-+ }
++#ifdef ENABLE_DEBUG
++ clock_t end = clock();
+
-+ // Append it to the list
-+ r = loc_country_list_append(countries, country);
-+ if (r) {
-+ PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
++ // Log how fast this has been
++ DEBUG(list->ctx, "Found network in %.4fms at %jd\n",
++ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
++#endif
+
-+ loc_country_list_unref(countries);
-+ loc_country_unref(country);
-+ return NULL;
-+ }
++ return i;
++ }
+
-+ loc_country_unref(country);
++ if (result > 0) {
++ lo = i + 1;
++ i++;
++ } else {
++ hi = i - 1;
+ }
++ }
+
-+ loc_database_enumerator_set_countries(enumerator, countries);
++ *found = 0;
+
-+ Py_DECREF(country_codes);
-+ loc_country_list_unref(countries);
- }
-
- // Set the ASN we are searching for
---
-2.20.1
-
-From 7af51f8a579c79714992a3e175036fb511139310 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Mon, 16 Nov 2020 15:20:50 +0000
-Subject: [PATCH 057/111] python: Only return country codes we want
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/export.py | 8 +++++++-
- 1 file changed, 7 insertions(+), 1 deletion(-)
-
-diff --git a/src/python/export.py b/src/python/export.py
-index be4a68e..5e7fe53 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -184,8 +184,14 @@ class Exporter(object):
-
- writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
-
-+ # Filter countries from special country codes
-+ country_codes = [
-+ country_code for country_code in countries if not country_code in flags.values()
-+ ]
++#ifdef ENABLE_DEBUG
++ clock_t end = clock();
+
- # Get all networks that match the family
-- networks = self.db.search_networks(family=family, flatten=True)
-+ networks = self.db.search_networks(family=family,
-+ country_codes=country_codes, flatten=True)
-
- # Walk through all networks
- for network in networks:
---
-2.20.1
-
-From bd1dc6bf6fe4ce40bf12e7426e283b31afd274e1 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Mon, 16 Nov 2020 15:25:15 +0000
-Subject: [PATCH 058/111] database: Filter flags in C
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/export.py | 16 +++++++++++-----
- 1 file changed, 11 insertions(+), 5 deletions(-)
-
-diff --git a/src/python/export.py b/src/python/export.py
-index 5e7fe53..739742f 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -29,7 +29,7 @@ import _location
- log = logging.getLogger("location.export")
- log.propagate = 1
-
--flags = {
-+FLAGS = {
- _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
- _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
- _location.NETWORK_FLAG_ANYCAST : "A3",
-@@ -186,12 +186,18 @@ class Exporter(object):
-
- # Filter countries from special country codes
- country_codes = [
-- country_code for country_code in countries if not country_code in flags.values()
-+ country_code for country_code in countries if not country_code in FLAGS.values()
- ]
-
-+ # Collect flags
-+ flags = 0
-+ for flag in FLAGS:
-+ if FLAGS[flag] in countries:
-+ flags |= flag
++ // Log how fast this has been
++ DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n",
++ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
++#endif
+
- # Get all networks that match the family
- networks = self.db.search_networks(family=family,
-- country_codes=country_codes, flatten=True)
-+ country_codes=country_codes, flags=flags, flatten=True)
-
- # Walk through all networks
- for network in networks:
-@@ -208,10 +214,10 @@ class Exporter(object):
- pass
-
- # Handle flags
-- for flag in flags:
-+ for flag in FLAGS:
- if network.has_flag(flag):
- # Fetch the "fake" country code
-- country = flags[flag]
-+ country = FLAGS[flag]
-
- try:
- writers[country].write(network)
---
-2.20.1
-
-From 84a2f0c2d9cbf8ae4225802c29ccba86561c77ed Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 17 Nov 2020 16:46:48 +0000
-Subject: [PATCH 059/111] as: Add list for easier processing
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- Makefile.am | 2 +
- src/as-list.c | 138 ++++++++++++++++++++++++++++++++++++++++++
- src/database.c | 29 +++++++--
- src/libloc.sym | 15 ++++-
- src/loc/as-list.h | 41 +++++++++++++
- src/loc/database.h | 5 +-
- src/python/database.c | 58 ++++++++++++++++--
- src/python/export.py | 2 +-
- 8 files changed, 275 insertions(+), 15 deletions(-)
- create mode 100644 src/as-list.c
- create mode 100644 src/loc/as-list.h
-
-diff --git a/Makefile.am b/Makefile.am
-index f4ca3c8..d0cc793 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -91,6 +91,7 @@ EXTRA_DIST += \
- pkginclude_HEADERS = \
- src/loc/libloc.h \
- src/loc/as.h \
-+ src/loc/as-list.h \
- src/loc/compat.h \
- src/loc/country.h \
- src/loc/country-list.h \
-@@ -109,6 +110,7 @@ lib_LTLIBRARIES = \
- src_libloc_la_SOURCES = \
- src/libloc.c \
- src/as.c \
-+ src/as-list.c \
- src/country.c \
- src/country-list.c \
- src/database.c \
-diff --git a/src/as-list.c b/src/as-list.c
-new file mode 100644
-index 0000000..7c69eb0
---- /dev/null
-+++ b/src/as-list.c
-@@ -0,0 +1,138 @@
-+/*
-+ libloc - A library to determine the location of someone on the Internet
++ return i;
++}
++
++LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
++ int found = 0;
++
++ off_t index = loc_network_list_find(list, network, &found);
++
++ // The network has been found on the list. Nothing to do.
++ if (found)
++ return 0;
++
++ DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n",
++ list, network, (intmax_t)index);
+
-+ Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
++ // Check if we have space left
++ if (list->size >= list->elements_size) {
++ int r = loc_network_list_grow(list, 64);
++ if (r)
++ return r;
++ }
+
-+ This library is free software; you can redistribute it and/or
-+ modify it under the terms of the GNU Lesser General Public
-+ License as published by the Free Software Foundation; either
-+ version 2.1 of the License, or (at your option) any later version.
++ // The list is now larger
++ list->size++;
+
-+ This library is distributed in the hope that it will be useful,
-+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ Lesser General Public License for more details.
-+*/
++ // Move all elements out of the way
++ for (unsigned int i = list->size - 1; i > index; i--)
++ list->elements[i] = list->elements[i - 1];
+
-+#include <errno.h>
-+#include <stdlib.h>
++ // Add the new element at the right place
++ list->elements[index] = loc_network_ref(network);
+
-+#include <loc/as.h>
-+#include <loc/as-list.h>
-+#include <loc/private.h>
++ return 0;
++}
+
-+struct loc_as_list {
-+ struct loc_ctx* ctx;
-+ int refcount;
++LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
++ // Return nothing when empty
++ if (loc_network_list_empty(list)) {
++ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
++ return NULL;
++ }
+
-+ struct loc_as* list[1024];
-+ size_t size;
-+ size_t max_size;
-+};
++ struct loc_network* network = list->elements[--list->size];
+
-+LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
-+ struct loc_as_list** list) {
-+ struct loc_as_list* l = calloc(1, sizeof(*l));
-+ if (!l)
-+ return -ENOMEM;
++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
+
-+ l->ctx = loc_ref(ctx);
-+ l->refcount = 1;
++ return network;
++}
++
++LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
++ // Return nothing when empty
++ if (loc_network_list_empty(list)) {
++ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
++ return NULL;
++ }
+
-+ // Do not allow this list to grow larger than this
-+ l->max_size = 1024;
++ struct loc_network* network = list->elements[0];
+
-+ DEBUG(l->ctx, "AS list allocated at %p\n", l);
-+ *list = l;
++ // Move all elements to the top of the stack
++ for (unsigned int i = 0; i < list->size - 1; i++) {
++ list->elements[i] = list->elements[i+1];
++ }
++
++ // The list is shorter now
++ --list->size;
++
++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
++
++ return network;
++}
++
++LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
++ int found = 0;
++
++ loc_network_list_find(list, network, &found);
++
++ return found;
++}
++
++LOC_EXPORT int loc_network_list_merge(
++ struct loc_network_list* self, struct loc_network_list* other) {
++ int r;
++
++ for (unsigned int i = 0; i < other->size; i++) {
++ r = loc_network_list_push(self, other->elements[i]);
++ if (r)
++ return r;
++ }
+
+ return 0;
+}
+diff --git a/src/network.c b/src/network.c
+index 366caa2..a6b679c 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -29,6 +29,7 @@
+ #include <loc/compat.h>
+ #include <loc/country.h>
+ #include <loc/network.h>
++#include <loc/network-list.h>
+ #include <loc/private.h>
+
+ struct loc_network {
+@@ -97,6 +98,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
+ return a;
+ }
+
++static struct in6_addr address_increment(const struct in6_addr* address) {
++ struct in6_addr a = *address;
+
-+LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
-+ list->refcount++;
++ for (int octet = 15; octet >= 0; octet--) {
++ if (a.s6_addr[octet] < 255) {
++ a.s6_addr[octet]++;
++ break;
++ } else {
++ a.s6_addr[octet] = 0;
++ }
++ }
+
-+ return list;
++ return a;
+}
+
-+static void loc_as_list_free(struct loc_as_list* list) {
-+ DEBUG(list->ctx, "Releasing AS list at %p\n", list);
+ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
+ struct in6_addr* address, unsigned int prefix) {
+ // Address cannot be unspecified
+@@ -160,9 +176,11 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
+ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
+ const char* address_string) {
+ struct in6_addr first_address;
+- unsigned int prefix = 0;
+ char* prefix_string;
+- int r = 1;
++ unsigned int prefix = 128;
++ int r = -EINVAL;
+
-+ loc_as_list_clear(list);
++ DEBUG(ctx, "Attempting to parse network %s\n", address_string);
+
+ // Make a copy of the string to work on it
+ char* buffer = strdup(address_string);
+@@ -171,29 +189,40 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
+ // Split address and prefix
+ address_string = strsep(&prefix_string, "/");
+
+- // Did we find a prefix?
++ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
+
-+ loc_unref(list->ctx);
-+ free(list);
++ // Parse the address
++ r = loc_parse_address(ctx, address_string, &first_address);
++ if (r) {
++ DEBUG(ctx, "The address could not be parsed\n");
++ goto FAIL;
++ }
++
++ // If a prefix was given, we will try to parse it
+ if (prefix_string) {
+ // Convert prefix to integer
+ prefix = strtol(prefix_string, NULL, 10);
+
+- if (prefix) {
+- // Parse the address
+- r = loc_parse_address(ctx, address_string, &first_address);
+-
+- // Map the prefix to IPv6 if needed
+- if (IN6_IS_ADDR_V4MAPPED(&first_address))
+- prefix += 96;
++ if (!prefix) {
++ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
++ goto FAIL;
+ }
++
++ // Map the prefix to IPv6 if needed
++ if (IN6_IS_ADDR_V4MAPPED(&first_address))
++ prefix += 96;
+ }
+
++FAIL:
+ // Free temporary buffer
+ free(buffer);
+
+- if (r == 0) {
+- r = loc_network_new(ctx, network, &first_address, prefix);
+- }
++ // Exit if the parsing was unsuccessful
++ if (r)
++ return r;
+
+- return r;
++ // Create a new network
++ return loc_network_new(ctx, network, &first_address, prefix);
+ }
+
+ LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
+@@ -281,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) {
+ return network->family;
+ }
+
++LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) {
++ switch (network->family) {
++ case AF_INET6:
++ return network->prefix;
++
++ case AF_INET:
++ return network->prefix - 96;
++ }
++
++ return 0;
+}
+
-+LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
-+ if (!list)
-+ return NULL;
+ static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) {
+ const size_t length = INET6_ADDRSTRLEN;
+
+@@ -314,10 +355,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc
+ return string;
+ }
+
++LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) {
++ return &network->first_address;
++}
+
-+ if (--list->refcount > 0)
-+ return list;
+ LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) {
+ return loc_network_format_address(network, &network->first_address);
+ }
+
++LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) {
++ return &network->last_address;
++}
++
+ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
+ return loc_network_format_address(network, &network->last_address);
+ }
+@@ -325,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
+ LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) {
+ // Address must be larger than the start address
+ if (in6_addr_cmp(&network->first_address, address) > 0)
+- return 1;
++ return 0;
+
+ // Address must be smaller than the last address
+ if (in6_addr_cmp(&network->last_address, address) < 0)
+- return 1;
++ return 0;
+
+ // The address is inside this network
+- return 0;
++ return 1;
+ }
+
+ LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) {
+@@ -392,20 +441,310 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
+ return loc_network_has_flag(network, flag);
+ }
+
+-LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
++LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
++ // Compare address
++ int r = in6_addr_cmp(&self->first_address, &other->first_address);
++ if (r)
++ return r;
++
++ // Compare prefix
++ if (self->prefix > other->prefix)
++ return 1;
++ else if (self->prefix < other->prefix)
++ return -1;
+
-+ loc_as_list_free(list);
-+ return NULL;
++ // Both networks are equal
++ return 0;
+}
+
-+LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
-+ return list->size;
-+}
++LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
++ // Either of the start addresses must be in the other subnet
++ if (loc_network_match_address(self, &other->first_address))
++ return 1;
+
-+LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
-+ return list->size == 0;
-+}
++ if (loc_network_match_address(other, &self->first_address))
++ return 1;
+
-+LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
-+ for (unsigned int i = 0; i < list->size; i++)
-+ loc_as_unref(list->list[i]);
-+}
++ // Or either of the end addresses is in the other subnet
++ if (loc_network_match_address(self, &other->last_address))
++ return 1;
+
-+LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
-+ // Check index
-+ if (index >= list->size)
-+ return NULL;
++ if (loc_network_match_address(other, &self->last_address))
++ return 1;
+
-+ return loc_as_ref(list->list[index]);
++ return 0;
+}
+
-+LOC_EXPORT int loc_as_list_append(
-+ struct loc_as_list* list, struct loc_as* as) {
-+ if (loc_as_list_contains(list, as))
++LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
++ // The prefix must be smaller (this avoids the more complex comparisons later)
++ if (self->prefix > other->prefix)
+ return 0;
+
-+ // Check if we have space left
-+ if (list->size == list->max_size) {
-+ ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
-+ return -ENOMEM;
-+ }
+ // If the start address of the other network is smaller than this network,
+ // it cannot be a subnet.
+- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
++ if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
+ return 0;
+
+ // If the end address of the other network is greater than this network,
+ // it cannot be a subnet.
+- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
++ if (in6_addr_cmp(&self->last_address, &other->last_address) < 0)
+ return 0;
+
+ return 1;
+ }
+
++LOC_EXPORT int loc_network_subnets(struct loc_network* network,
++ struct loc_network** subnet1, struct loc_network** subnet2) {
++ int r;
++ *subnet1 = NULL;
++ *subnet2 = NULL;
+
-+ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
++ // New prefix length
++ unsigned int prefix = network->prefix + 1;
++
++ // Check if the new prefix is valid
++ if (valid_prefix(&network->first_address, prefix))
++ return -1;
+
-+ list->list[list->size++] = loc_as_ref(as);
++ // Create the first half of the network
++ r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix);
++ if (r)
++ return r;
+
-+ return 0;
-+}
++ // The next subnet starts after the first one
++ struct in6_addr first_address = address_increment(&(*subnet1)->last_address);
+
-+LOC_EXPORT int loc_as_list_contains(
-+ struct loc_as_list* list, struct loc_as* as) {
-+ for (unsigned int i = 0; i < list->size; i++) {
-+ if (loc_as_cmp(as, list->list[i]) == 0)
-+ return 1;
++ // Create the second half of the network
++ r = loc_network_new(network->ctx, subnet2, &first_address, prefix);
++ if (r)
++ return r;
++
++ // Copy country code
++ const char* country_code = loc_network_get_country_code(network);
++ if (country_code) {
++ loc_network_set_country_code(*subnet1, country_code);
++ loc_network_set_country_code(*subnet2, country_code);
++ }
++
++ // Copy ASN
++ uint32_t asn = loc_network_get_asn(network);
++ if (asn) {
++ loc_network_set_asn(*subnet1, asn);
++ loc_network_set_asn(*subnet2, asn);
+ }
+
++ // Copy flags
++ loc_network_set_flag(*subnet1, network->flags);
++ loc_network_set_flag(*subnet2, network->flags);
++
+ return 0;
+}
+
-+LOC_EXPORT int loc_as_list_contains_number(
-+ struct loc_as_list* list, uint32_t number) {
-+ struct loc_as* as;
++static int __loc_network_exclude(struct loc_network* network,
++ struct loc_network* other, struct loc_network_list* list) {
++ struct loc_network* subnet1 = NULL;
++ struct loc_network* subnet2 = NULL;
+
-+ int r = loc_as_new(list->ctx, &as, number);
++ int r = loc_network_subnets(network, &subnet1, &subnet2);
+ if (r)
-+ return -1;
++ goto ERROR;
+
-+ r = loc_as_list_contains(list, as);
-+ loc_as_unref(as);
++ if (loc_network_cmp(other, subnet1) == 0) {
++ r = loc_network_list_push(list, subnet2);
++ if (r)
++ goto ERROR;
+
-+ return r;
-+}
-diff --git a/src/database.c b/src/database.c
-index 29823b2..51cb5cd 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -38,6 +38,7 @@
-
- #include <loc/libloc.h>
- #include <loc/as.h>
-+#include <loc/as-list.h>
- #include <loc/compat.h>
- #include <loc/country.h>
- #include <loc/country-list.h>
-@@ -101,7 +102,7 @@ struct loc_database_enumerator {
- // Search string
- char* string;
- struct loc_country_list* countries;
-- uint32_t asn;
-+ struct loc_as_list* asns;
- enum loc_network_flags flags;
- int family;
-
-@@ -1036,9 +1037,20 @@ LOC_EXPORT int loc_database_enumerator_set_countries(
- return 0;
- }
-
--LOC_EXPORT int loc_database_enumerator_set_asn(
-- struct loc_database_enumerator* enumerator, unsigned int asn) {
-- enumerator->asn = asn;
-+LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
-+ struct loc_database_enumerator* enumerator) {
-+ if (!enumerator->asns)
-+ return NULL;
++ } else if (loc_network_cmp(other, subnet2) == 0) {
++ r = loc_network_list_push(list, subnet1);
++ if (r)
++ goto ERROR;
+
-+ return loc_as_list_ref(enumerator->asns);
-+}
++ } else if (loc_network_is_subnet(subnet1, other)) {
++ r = loc_network_list_push(list, subnet2);
++ if (r)
++ goto ERROR;
+
-+LOC_EXPORT int loc_database_enumerator_set_asns(
-+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
-+ if (enumerator->asns)
-+ loc_as_list_unref(enumerator->asns);
++ r = __loc_network_exclude(subnet1, other, list);
++ if (r)
++ goto ERROR;
+
-+ enumerator->asns = loc_as_list_ref(asns);
-
- return 0;
- }
-@@ -1123,6 +1135,12 @@ static int loc_network_match_countries(struct loc_network* network, struct loc_c
- return loc_country_list_contains_code(countries, country_code);
- }
-
-+static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
-+ uint32_t asn = loc_network_get_asn(network);
++ } else if (loc_network_is_subnet(subnet2, other)) {
++ r = loc_network_list_push(list, subnet1);
++ if (r)
++ goto ERROR;
+
-+ return loc_as_list_contains_number(asns, asn);
-+}
++ r = __loc_network_exclude(subnet2, other, list);
++ if (r)
++ goto ERROR;
+
- static int loc_database_enumerator_filter_network(
- struct loc_database_enumerator* enumerator, struct loc_network* network) {
- // Skip if the family does not match
-@@ -1134,8 +1152,7 @@ static int loc_database_enumerator_filter_network(
- return 1;
-
- // Skip if the ASN does not match
-- if (enumerator->asn &&
-- !loc_network_match_asn(network, enumerator->asn))
-+ if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
- return 1;
-
- // Skip if flags do not match
-diff --git a/src/libloc.sym b/src/libloc.sym
-index 40e9f88..53273cd 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -37,6 +37,18 @@ global:
- loc_as_set_name;
- loc_as_unref;
-
-+ # AS List
-+ loc_as_list_append;
-+ loc_as_list_clear;
-+ loc_as_list_contains;
-+ loc_as_list_contains_number;
-+ loc_as_list_empty;
-+ loc_as_list_get;
-+ loc_as_list_new;
-+ loc_as_list_ref;
-+ loc_as_list_size;
-+ loc_as_list_unref;
++ } else {
++ ERROR(network->ctx, "We should never get here\n");
++ r = 1;
++ goto ERROR;
++ }
+
- # Country
- loc_country_cmp;
- loc_country_code_is_valid;
-@@ -78,13 +90,14 @@ global:
- loc_database_verify;
-
- # Database Enumerator
-+ loc_database_enumerator_get_asns;
- loc_database_enumerator_get_countries;
- loc_database_enumerator_new;
- loc_database_enumerator_next_as;
- loc_database_enumerator_next_country;
- loc_database_enumerator_next_network;
- loc_database_enumerator_ref;
-- loc_database_enumerator_set_asn;
-+ loc_database_enumerator_set_asns;
- loc_database_enumerator_set_countries;
- loc_database_enumerator_set_family;
- loc_database_enumerator_set_flag;
-diff --git a/src/loc/as-list.h b/src/loc/as-list.h
-new file mode 100644
-index 0000000..7b5c4e8
---- /dev/null
-+++ b/src/loc/as-list.h
-@@ -0,0 +1,41 @@
-+/*
-+ libloc - A library to determine the location of someone on the Internet
++ERROR:
++ if (subnet1)
++ loc_network_unref(subnet1);
+
-+ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
++ if (subnet2)
++ loc_network_unref(subnet2);
+
-+ This library is free software; you can redistribute it and/or
-+ modify it under the terms of the GNU Lesser General Public
-+ License as published by the Free Software Foundation; either
-+ version 2.1 of the License, or (at your option) any later version.
++ return r;
++}
+
-+ This library is distributed in the hope that it will be useful,
-+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ Lesser General Public License for more details.
-+*/
++static int __loc_network_exclude_to_list(struct loc_network* self,
++ struct loc_network* other, struct loc_network_list* list) {
++ // Other must be a subnet of self
++ if (!loc_network_is_subnet(self, other)) {
++ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
++
++ // Exit silently
++ return 0;
++ }
++
++ // We cannot perform this operation if both networks equal
++ if (loc_network_cmp(self, other) == 0) {
++ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
++
++ // Exit silently
++ return 0;
++ }
+
-+#ifndef LIBLOC_AS_LIST_H
-+#define LIBLOC_AS_LIST_H
++ return __loc_network_exclude(self, other, list);
++}
+
-+#include <loc/as.h>
-+#include <loc/libloc.h>
++LOC_EXPORT struct loc_network_list* loc_network_exclude(
++ struct loc_network* self, struct loc_network* other) {
++ struct loc_network_list* list;
+
-+struct loc_as_list;
++#ifdef ENABLE_DEBUG
++ char* n1 = loc_network_str(self);
++ char* n2 = loc_network_str(other);
+
-+int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
-+struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
-+struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
++ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
+
-+size_t loc_as_list_size(struct loc_as_list* list);
-+int loc_as_list_empty(struct loc_as_list* list);
-+void loc_as_list_clear(struct loc_as_list* list);
++ free(n1);
++ free(n2);
++#endif
+
-+struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
-+int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
++ // Create a new list with the result
++ int r = loc_network_list_new(self->ctx, &list);
++ if (r) {
++ ERROR(self->ctx, "Could not create network list: %d\n", r);
+
-+int loc_as_list_contains(
-+ struct loc_as_list* list, struct loc_as* as);
-+int loc_as_list_contains_number(
-+ struct loc_as_list* list, uint32_t number);
++ return NULL;
++ }
+
-+#endif
-diff --git a/src/loc/database.h b/src/loc/database.h
-index 246e5c5..70801f0 100644
---- a/src/loc/database.h
-+++ b/src/loc/database.h
-@@ -70,7 +70,10 @@ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerato
- struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
- int loc_database_enumerator_set_countries(
- struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
--int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
-+struct loc_as_list* loc_database_enumerator_get_asns(
-+ struct loc_database_enumerator* enumerator);
-+int loc_database_enumerator_set_asns(
-+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
- int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
- int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
- int loc_database_enumerator_next_as(
-diff --git a/src/python/database.c b/src/python/database.c
-index e6f6f37..38a804c 100644
---- a/src/python/database.c
-+++ b/src/python/database.c
-@@ -17,6 +17,8 @@
- #include <Python.h>
-
- #include <loc/libloc.h>
-+#include <loc/as.h>
-+#include <loc/as-list.h>
- #include <loc/database.h>
-
- #include "locationmodule.h"
-@@ -258,15 +260,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
- }
-
- static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
-- char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
-+ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
- PyObject* country_codes = NULL;
-- unsigned int asn = 0;
-+ PyObject* asn_list = NULL;
- int flags = 0;
- int family = 0;
- int flatten = 0;
-
-- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
-- &PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
-+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
-+ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
- return NULL;
-
- struct loc_database_enumerator* enumerator;
-@@ -330,13 +332,57 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
- }
-
- // Set the ASN we are searching for
-- if (asn) {
-- r = loc_database_enumerator_set_asn(enumerator, asn);
-+ if (asn_list) {
-+ struct loc_as_list* asns;
-+ r = loc_as_list_new(loc_ctx, &asns);
-+ if (r) {
-+ PyErr_SetString(PyExc_SystemError, "Could not create AS list");
-+ return NULL;
-+ }
++ r = __loc_network_exclude_to_list(self, other, list);
++ if (r) {
++ loc_network_list_unref(list);
+
-+ for (unsigned int i = 0; i < PyList_Size(asn_list); i++) {
-+ PyObject* item = PyList_GetItem(asn_list, i);
++ return NULL;
++ }
+
-+ if (!PyLong_Check(item)) {
-+ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
++ // Return the result
++ return list;
++}
+
-+ loc_as_list_unref(asns);
-+ return NULL;
-+ }
++LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
++ struct loc_network* network, struct loc_network_list* list) {
++ struct loc_network_list* to_check;
+
-+ unsigned long number = PyLong_AsLong(item);
-
-+ struct loc_as* as;
-+ r = loc_as_new(loc_ctx, &as, number);
-+ if (r) {
-+ PyErr_SetString(PyExc_SystemError, "Could not create AS");
++ // Create a new list with all networks to look at
++ int r = loc_network_list_new(network->ctx, &to_check);
++ if (r)
++ return NULL;
+
-+ loc_as_list_unref(asns);
-+ loc_as_unref(as);
-+ return NULL;
-+ }
++ struct loc_network* subnet = NULL;
++ struct loc_network_list* subnets = NULL;
+
-+ r = loc_as_list_append(asns, as);
++ for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
++ subnet = loc_network_list_get(list, i);
++
++ // Find all excluded networks
++ if (!loc_network_list_contains(to_check, subnet)) {
++ r = __loc_network_exclude_to_list(network, subnet, to_check);
+ if (r) {
-+ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
++ loc_network_list_unref(to_check);
++ loc_network_unref(subnet);
+
-+ loc_as_list_unref(asns);
-+ loc_as_unref(as);
+ return NULL;
+ }
-+
-+ loc_as_unref(as);
+ }
+
-+ r = loc_database_enumerator_set_asns(enumerator, asns);
- if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
-+
-+ loc_as_list_unref(asns);
- return NULL;
- }
-+
-+ loc_as_list_unref(asns);
- }
-
- // Set the flags we are searching for
-diff --git a/src/python/export.py b/src/python/export.py
-index 739742f..f675eb3 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -197,7 +197,7 @@ class Exporter(object):
-
- # Get all networks that match the family
- networks = self.db.search_networks(family=family,
-- country_codes=country_codes, flags=flags, flatten=True)
-+ country_codes=country_codes, asns=asns, flags=flags, flatten=True)
-
- # Walk through all networks
- for network in networks:
---
-2.20.1
-
-From 50120b991fc2fa4b7813096de87b42d700faf3e6 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 17 Nov 2020 16:56:43 +0000
-Subject: [PATCH 060/111] database: Simplify network matching code
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 16 ++++++++--------
- 1 file changed, 8 insertions(+), 8 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index 51cb5cd..1a354f6 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
- return 0;
- }
-
--static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
-- const char* country_code = loc_network_get_country_code(network);
--
-- return loc_country_list_contains_code(countries, country_code);
--}
--
- static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
- uint32_t asn = loc_network_get_asn(network);
-
-@@ -1148,8 +1142,14 @@ static int loc_database_enumerator_filter_network(
- return 1;
-
- // Skip if the country code does not match
-- if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
-- return 1;
-+ if (enumerator->countries) {
-+ if (!loc_country_list_empty(enumerator->countries)) {
-+ const char* country_code = loc_network_get_country_code(network);
-+
-+ if (!loc_country_list_contains_code(enumerator->countries, country_code))
-+ return 1;
-+ }
++ // Cleanup
++ loc_network_unref(subnet);
+ }
-
- // Skip if the ASN does not match
- if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
---
-2.20.1
-
-From c1a36c943181da5cd2aef589a972d5027e529eb8 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 17 Nov 2020 16:58:55 +0000
-Subject: [PATCH 061/111] database: Simplify AS matching code
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 16 ++++++++--------
- 1 file changed, 8 insertions(+), 8 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index 1a354f6..be93e00 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
- return 0;
- }
-
--static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
-- uint32_t asn = loc_network_get_asn(network);
--
-- return loc_as_list_contains_number(asns, asn);
--}
--
- static int loc_database_enumerator_filter_network(
- struct loc_database_enumerator* enumerator, struct loc_network* network) {
- // Skip if the family does not match
-@@ -1152,8 +1146,14 @@ static int loc_database_enumerator_filter_network(
- }
-
- // Skip if the ASN does not match
-- if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
-- return 1;
-+ if (enumerator->asns) {
-+ if (!loc_as_list_empty(enumerator->asns)) {
-+ uint32_t asn = loc_network_get_asn(network);
+
-+ if (!loc_as_list_contains_number(enumerator->asns, asn))
-+ return 1;
-+ }
++ r = loc_network_list_new(network->ctx, &subnets);
++ if (r) {
++ loc_network_list_unref(to_check);
++ return NULL;
+ }
-
- // Skip if flags do not match
- if (enumerator->flags &&
---
-2.20.1
-
-From d5205091f9cc1ff987e483325d48696459df08d8 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 17 Nov 2020 17:50:17 +0000
-Subject: [PATCH 062/111] countries: Make list grow dynamically
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/country-list.c | 38 ++++++++++++++++++++++++++------------
- 1 file changed, 26 insertions(+), 12 deletions(-)
-
-diff --git a/src/country-list.c b/src/country-list.c
-index ae0d71a..1ce2d06 100644
---- a/src/country-list.c
-+++ b/src/country-list.c
-@@ -25,11 +25,27 @@ struct loc_country_list {
- struct loc_ctx* ctx;
- int refcount;
-
-- struct loc_country* list[1024];
-+ struct loc_country** elements;
-+ size_t elements_size;
+
- size_t size;
-- size_t max_size;
- };
-
-+static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
-+ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
-+ list, size, list->elements_size + size);
++ off_t smallest_subnet = 0;
+
-+ struct loc_country** elements = reallocarray(list->elements,
-+ list->elements_size + size, sizeof(*list->elements));
-+ if (!elements)
-+ return -errno;
++ while (!loc_network_list_empty(to_check)) {
++ struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check);
+
-+ list->elements = elements;
-+ list->elements_size += size;
++ // Check whether the subnet to check is part of the input list
++ if (loc_network_list_contains(list, subnet_to_check)) {
++ loc_network_unref(subnet_to_check);
++ continue;
++ }
+
-+ return 0;
-+}
++ // Marks whether this subnet passed all checks
++ int passed = 1;
+
- LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
- struct loc_country_list** list) {
- struct loc_country_list* l = calloc(1, sizeof(*l));
-@@ -39,9 +55,6 @@ LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
- l->ctx = loc_ref(ctx);
- l->refcount = 1;
-
-- // Do not allow this list to grow larger than this
-- l->max_size = 1024;
--
- DEBUG(l->ctx, "Country list allocated at %p\n", l);
- *list = l;
-
-@@ -84,7 +97,7 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
-
- LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
- for (unsigned int i = 0; i < list->size; i++)
-- loc_country_unref(list->list[i]);
-+ loc_country_unref(list->elements[i]);
- }
-
- LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
-@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* lis
- if (index >= list->size)
- return NULL;
-
-- return loc_country_ref(list->list[index]);
-+ return loc_country_ref(list->elements[index]);
- }
-
- LOC_EXPORT int loc_country_list_append(
-@@ -101,14 +114,15 @@ LOC_EXPORT int loc_country_list_append(
- return 0;
-
- // Check if we have space left
-- if (list->size == list->max_size) {
-- ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
-- return -ENOMEM;
-+ if (list->size >= list->elements_size) {
-+ int r = loc_country_list_grow(list, 64);
-+ if (r)
-+ return r;
- }
-
- DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
-
-- list->list[list->size++] = loc_country_ref(country);
-+ list->elements[list->size++] = loc_country_ref(country);
-
- return 0;
- }
-@@ -116,7 +130,7 @@ LOC_EXPORT int loc_country_list_append(
- LOC_EXPORT int loc_country_list_contains(
- struct loc_country_list* list, struct loc_country* country) {
- for (unsigned int i = 0; i < list->size; i++) {
-- if (loc_country_cmp(country, list->list[i]) == 0)
-+ if (loc_country_cmp(country, list->elements[i]) == 0)
- return 1;
- }
-
---
-2.20.1
-
-From 3b44e4211371d2103f89ba8f056b15edb7778fac Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 17 Nov 2020 17:55:51 +0000
-Subject: [PATCH 063/111] networks: Make list grow dynamically
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network-list.c | 60 ++++++++++++++++++++++++++++------------------
- 1 file changed, 37 insertions(+), 23 deletions(-)
-
-diff --git a/src/network-list.c b/src/network-list.c
-index 1f6e80e..4912c02 100644
---- a/src/network-list.c
-+++ b/src/network-list.c
-@@ -25,11 +25,27 @@ struct loc_network_list {
- struct loc_ctx* ctx;
- int refcount;
-
-- struct loc_network* list[1024];
-+ struct loc_network** elements;
-+ size_t elements_size;
++ for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) {
++ subnet = loc_network_list_get(list, i);
++
++ // Drop this subnet if is a subnet of another subnet
++ if (loc_network_is_subnet(subnet, subnet_to_check)) {
++ passed = 0;
++ loc_network_unref(subnet);
++ break;
++ }
+
- size_t size;
-- size_t max_size;
- };
-
-+static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
-+ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
-+ list, size, list->elements_size + size);
++ // Break it down if it overlaps
++ if (loc_network_overlaps(subnet, subnet_to_check)) {
++ passed = 0;
+
-+ struct loc_network** elements = reallocarray(list->elements,
-+ list->elements_size + size, sizeof(*list->elements));
-+ if (!elements)
-+ return -errno;
++ __loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
+
-+ list->elements = elements;
-+ list->elements_size += size;
++ loc_network_unref(subnet);
++ break;
++ }
+
-+ return 0;
-+}
++ // If the subnet is strictly greater, we do not need to continue the search
++ r = loc_network_cmp(subnet, subnet_to_check);
++ if (r > 0) {
++ loc_network_unref(subnet);
++ break;
+
- LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
- struct loc_network_list** list) {
- struct loc_network_list* l = calloc(1, sizeof(*l));
-@@ -39,9 +55,6 @@ LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
- l->ctx = loc_ref(ctx);
- l->refcount = 1;
-
-- // Do not allow this list to grow larger than this
-- l->max_size = 1024;
--
- DEBUG(l->ctx, "Network list allocated at %p\n", l);
- *list = l;
- return 0;
-@@ -57,7 +70,7 @@ static void loc_network_list_free(struct loc_network_list* list) {
- DEBUG(list->ctx, "Releasing network list at %p\n", list);
-
- for (unsigned int i = 0; i < list->size; i++)
-- loc_network_unref(list->list[i]);
-+ loc_network_unref(list->elements[i]);
-
- loc_unref(list->ctx);
- free(list);
-@@ -84,7 +97,7 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
-
- LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
- for (unsigned int i = 0; i < list->size; i++)
-- loc_network_unref(list->list[i]);
-+ loc_network_unref(list->elements[i]);
-
- list->size = 0;
- }
-@@ -94,7 +107,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
- char* s;
-
- for (unsigned int i = 0; i < list->size; i++) {
-- network = list->list[i];
-+ network = list->elements[i];
-
- s = loc_network_str(network);
-
-@@ -108,7 +121,7 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
- if (index >= list->size)
- return NULL;
-
-- return loc_network_ref(list->list[index]);
-+ return loc_network_ref(list->elements[index]);
- }
-
- LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
-@@ -117,14 +130,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
- return 0;
-
- // Check if we have space left
-- if (list->size == list->max_size) {
-- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
-- return -ENOMEM;
-+ if (list->size >= list->elements_size) {
-+ int r = loc_network_list_grow(list, 64);
-+ if (r)
-+ return r;
- }
-
- DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
-
-- list->list[list->size++] = loc_network_ref(network);
-+ list->elements[list->size++] = loc_network_ref(network);
-
- return 0;
- }
-@@ -136,7 +150,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
- return NULL;
- }
-
-- struct loc_network* network = list->list[--list->size];
-+ struct loc_network* network = list->elements[--list->size];
-
- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
-
-@@ -150,11 +164,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
- return NULL;
- }
-
-- struct loc_network* network = list->list[0];
-+ struct loc_network* network = list->elements[0];
-
- // Move all elements to the top of the stack
- for (unsigned int i = 0; i < --list->size; i++) {
-- list->list[i] = list->list[i+1];
-+ list->elements[i] = list->elements[i+1];
- }
-
- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
-@@ -164,7 +178,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
-
- LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
- for (unsigned int i = 0; i < list->size; i++) {
-- if (loc_network_eq(list->list[i], network))
-+ if (loc_network_eq(list->elements[i], network))
- return 1;
- }
-
-@@ -176,11 +190,11 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
- if (i1 >= list->size || i2 >= list->size)
- return;
-
-- struct loc_network* network1 = list->list[i1];
-- struct loc_network* network2 = list->list[i2];
-+ struct loc_network* network1 = list->elements[i1];
-+ struct loc_network* network2 = list->elements[i2];
-
-- list->list[i1] = network2;
-- list->list[i2] = network1;
-+ list->elements[i1] = network2;
-+ list->elements[i2] = network1;
- }
-
- LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
-@@ -200,7 +214,7 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
- swapped = 0;
-
- for (unsigned int i = 1; i < n; i++) {
-- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
-+ if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) {
- loc_network_list_swap(list, i-1, i);
- swapped = 1;
- }
-@@ -215,7 +229,7 @@ LOC_EXPORT int loc_network_list_merge(
- int r;
-
- for (unsigned int i = 0; i < other->size; i++) {
-- r = loc_network_list_push(self, other->list[i]);
-+ r = loc_network_list_push(self, other->elements[i]);
- if (r)
- return r;
- }
---
-2.20.1
-
-From 1a415f8c555f4fe9a68eb2a897c4a1fc0d33db25 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 17 Nov 2020 17:57:55 +0000
-Subject: [PATCH 064/111] as: Make lists grow dynamically
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/as-list.c | 38 ++++++++++++++++++++++++++------------
- 1 file changed, 26 insertions(+), 12 deletions(-)
-
-diff --git a/src/as-list.c b/src/as-list.c
-index 7c69eb0..17de23e 100644
---- a/src/as-list.c
-+++ b/src/as-list.c
-@@ -25,11 +25,27 @@ struct loc_as_list {
- struct loc_ctx* ctx;
- int refcount;
-
-- struct loc_as* list[1024];
-+ struct loc_as** elements;
-+ size_t elements_size;
++ // If it is strictly smaller, we can continue the search from here next
++ // time because all networks that are to be checked can only be larger
++ // than this one.
++ } else if (r < 0) {
++ smallest_subnet = i;
++ }
+
- size_t size;
-- size_t max_size;
- };
-
-+static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
-+ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
-+ list, size, list->elements_size + size);
++ loc_network_unref(subnet);
++ }
+
-+ struct loc_as** elements = reallocarray(list->elements,
-+ list->elements_size + size, sizeof(*list->elements));
-+ if (!elements)
-+ return -errno;
++ if (passed) {
++ r = loc_network_list_push(subnets, subnet_to_check);
++ }
+
-+ list->elements = elements;
-+ list->elements_size += size;
++ loc_network_unref(subnet_to_check);
++ }
+
-+ return 0;
++ loc_network_list_unref(to_check);
++
++ return subnets;
+}
+
- LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
- struct loc_as_list** list) {
- struct loc_as_list* l = calloc(1, sizeof(*l));
-@@ -39,9 +55,6 @@ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
- l->ctx = loc_ref(ctx);
- l->refcount = 1;
-
-- // Do not allow this list to grow larger than this
-- l->max_size = 1024;
--
- DEBUG(l->ctx, "AS list allocated at %p\n", l);
- *list = l;
-
-@@ -84,7 +97,7 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
-
- LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
- for (unsigned int i = 0; i < list->size; i++)
-- loc_as_unref(list->list[i]);
-+ loc_as_unref(list->elements[i]);
- }
-
- LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
-@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index
- if (index >= list->size)
- return NULL;
-
-- return loc_as_ref(list->list[index]);
-+ return loc_as_ref(list->elements[index]);
- }
-
- LOC_EXPORT int loc_as_list_append(
-@@ -101,14 +114,15 @@ LOC_EXPORT int loc_as_list_append(
- return 0;
-
- // Check if we have space left
-- if (list->size == list->max_size) {
-- ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
-- return -ENOMEM;
-+ if (list->size >= list->elements_size) {
-+ int r = loc_as_list_grow(list, 64);
-+ if (r)
-+ return r;
- }
-
- DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
-
-- list->list[list->size++] = loc_as_ref(as);
-+ list->elements[list->size++] = loc_as_ref(as);
+ LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
+ // Add country code
+ loc_country_code_copy(dbobj->country_code, network->country_code);
+@@ -474,7 +813,7 @@ struct loc_network_tree_node {
+ struct loc_network* network;
+ };
+-LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
++int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
+ struct loc_network_tree* t = calloc(1, sizeof(*t));
+ if (!t)
+ return -ENOMEM;
+@@ -494,7 +833,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
return 0;
}
-@@ -116,7 +130,7 @@ LOC_EXPORT int loc_as_list_append(
- LOC_EXPORT int loc_as_list_contains(
- struct loc_as_list* list, struct loc_as* as) {
- for (unsigned int i = 0; i < list->size; i++) {
-- if (loc_as_cmp(as, list->list[i]) == 0)
-+ if (loc_as_cmp(as, list->elements[i]) == 0)
- return 1;
- }
-
---
-2.20.1
-
-From e6592434ee7836507c1f436ec3b0db3bc81a81b9 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 17 Nov 2020 18:13:49 +0000
-Subject: [PATCH 065/111] export: Change back to use Network objects
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/export.py | 4 +---
- 1 file changed, 1 insertion(+), 3 deletions(-)
-
-diff --git a/src/python/export.py b/src/python/export.py
-index f675eb3..67e437f 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -144,9 +144,7 @@ class XTGeoIPOutputWriter(OutputWriter):
- def _write_network(self, network):
- for address in (network.first_address, network.last_address):
- # Convert this into a string of bits
-- bytes = socket.inet_pton(
-- socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
-- )
-+ bytes = socket.inet_pton(network.family, address)
-
- self.f.write(bytes)
-
---
-2.20.1
-
-From 248f5e0419f2349253b8ea96e477c15649fe2173 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 17 Nov 2020 18:14:15 +0000
-Subject: [PATCH 066/111] Actually clear all lists
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/as-list.c | 8 ++++++++
- src/country-list.c | 8 ++++++++
- src/network-list.c | 6 ++++++
- 3 files changed, 22 insertions(+)
-
-diff --git a/src/as-list.c b/src/as-list.c
-index 17de23e..76620c7 100644
---- a/src/as-list.c
-+++ b/src/as-list.c
-@@ -96,8 +96,16 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
- }
-
- LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
-+ if (!list->elements)
-+ return;
-+
- for (unsigned int i = 0; i < list->size; i++)
- loc_as_unref(list->elements[i]);
-+
-+ free(list->elements);
-+ list->elements_size = 0;
-+
-+ list->size = 0;
- }
- LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
-diff --git a/src/country-list.c b/src/country-list.c
-index 1ce2d06..1c49c47 100644
---- a/src/country-list.c
-+++ b/src/country-list.c
-@@ -96,8 +96,16 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
+-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
++struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
+ return loc_network_tree_node_ref(tree->root);
}
- LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
-+ if (!list->elements)
-+ return;
-+
- for (unsigned int i = 0; i < list->size; i++)
- loc_country_unref(list->elements[i]);
-+
-+ free(list->elements);
-+ list->elements_size = 0;
-+
-+ list->size = 0;
+@@ -566,7 +905,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
+ return 0;
}
- LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
-diff --git a/src/network-list.c b/src/network-list.c
-index 4912c02..9cb4547 100644
---- a/src/network-list.c
-+++ b/src/network-list.c
-@@ -96,9 +96,15 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
+-LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
++int loc_network_tree_walk(struct loc_network_tree* tree,
+ int(*filter_callback)(struct loc_network* network, void* data),
+ int(*callback)(struct loc_network* network, void* data), void* data) {
+ return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
+@@ -581,7 +920,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
+ free(tree);
}
- LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
-+ if (!list->elements)
-+ return;
-+
- for (unsigned int i = 0; i < list->size; i++)
- loc_network_unref(list->elements[i]);
+-LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
++struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
+ if (--tree->refcount > 0)
+ return tree;
-+ free(list->elements);
-+ list->elements_size = 0;
-+
- list->size = 0;
+@@ -602,13 +941,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
+ return 0;
}
---
-2.20.1
-
-From c98ebf8aae2aa141193db52cd9429b1ded5b09c4 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 17 Nov 2020 18:34:51 +0000
-Subject: [PATCH 067/111] database: Do not clean up python list
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/database.c | 9 +++++++--
- 1 file changed, 7 insertions(+), 2 deletions(-)
-
-diff --git a/src/python/database.c b/src/python/database.c
-index 38a804c..ed22275 100644
---- a/src/python/database.c
-+++ b/src/python/database.c
-@@ -325,9 +325,14 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
- loc_country_unref(country);
- }
-
-- loc_database_enumerator_set_countries(enumerator, countries);
-+ r = loc_database_enumerator_set_countries(enumerator, countries);
-+ if (r) {
-+ PyErr_SetFromErrno(PyExc_SystemError);
-+
-+ loc_as_list_unref(countries);
-+ return NULL;
-+ }
-
-- Py_DECREF(country_codes);
- loc_country_list_unref(countries);
- }
-
---
-2.20.1
-
-From 5470d06cb59027f4e04b6d576763dbf7f1093fde Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 17 Nov 2020 19:01:04 +0000
-Subject: [PATCH 068/111] database: Free filter lists in enumerator
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 6 ++++++
- 1 file changed, 6 insertions(+)
-
-diff --git a/src/database.c b/src/database.c
-index be93e00..ca35fe1 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -950,6 +950,12 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera
- if (enumerator->string)
- free(enumerator->string);
-
-+ if (enumerator->countries)
-+ loc_country_list_unref(enumerator->countries);
-+
-+ if (enumerator->asns)
-+ loc_as_list_unref(enumerator->asns);
-+
- // Free network search
- free(enumerator->networks_visited);
-
---
-2.20.1
-
-From e0e96878d3df51c4a265d51d088005dedf9335e3 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 18 Nov 2020 13:18:52 +0000
-Subject: [PATCH 069/111] database: Add debug output to filtering
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 29 ++++++++++++++++-------------
- 1 file changed, 16 insertions(+), 13 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index ca35fe1..83dd752 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1138,33 +1138,36 @@ static int loc_database_enumerator_stack_push_node(
- static int loc_database_enumerator_filter_network(
- struct loc_database_enumerator* enumerator, struct loc_network* network) {
- // Skip if the family does not match
-- if (enumerator->family && loc_network_address_family(network) != enumerator->family)
-+ if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
-+ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
- return 1;
-+ }
-
- // Skip if the country code does not match
-- if (enumerator->countries) {
-- if (!loc_country_list_empty(enumerator->countries)) {
-- const char* country_code = loc_network_get_country_code(network);
-+ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
-+ const char* country_code = loc_network_get_country_code(network);
-
-- if (!loc_country_list_contains_code(enumerator->countries, country_code))
-- return 1;
-+ if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
-+ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
-+ return 1;
- }
- }
-
- // Skip if the ASN does not match
-- if (enumerator->asns) {
-- if (!loc_as_list_empty(enumerator->asns)) {
-- uint32_t asn = loc_network_get_asn(network);
-+ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
-+ uint32_t asn = loc_network_get_asn(network);
-
-- if (!loc_as_list_contains_number(enumerator->asns, asn))
-- return 1;
-+ if (!loc_as_list_contains_number(enumerator->asns, asn)) {
-+ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
-+ return 1;
- }
- }
+-LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
++int loc_network_tree_dump(struct loc_network_tree* tree) {
+ DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
- // Skip if flags do not match
-- if (enumerator->flags &&
-- !loc_network_match_flag(network, enumerator->flags))
-+ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
-+ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
- return 1;
-+ }
+ return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
+ }
+
+-LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
++int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
+ DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
- // Do not filter
+ struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
+@@ -639,7 +978,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
return 0;
---
-2.20.1
-
-From bce0c9295ff8ff9488f24babe01ce851228d0b1e Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 18 Nov 2020 13:19:04 +0000
-Subject: [PATCH 070/111] export: Remove filtering for flags
-
-The filter is an AND filter and if we set the flags from
-the special country codes, we won't get back much.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/export.py | 8 +-------
- 1 file changed, 1 insertion(+), 7 deletions(-)
-
-diff --git a/src/python/export.py b/src/python/export.py
-index 67e437f..4219957 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -187,15 +187,9 @@ class Exporter(object):
- country_code for country_code in countries if not country_code in FLAGS.values()
- ]
-
-- # Collect flags
-- flags = 0
-- for flag in FLAGS:
-- if FLAGS[flag] in countries:
-- flags |= flag
--
- # Get all networks that match the family
- networks = self.db.search_networks(family=family,
-- country_codes=country_codes, asns=asns, flags=flags, flatten=True)
-+ country_codes=country_codes, asns=asns, flatten=True)
+ }
- # Walk through all networks
- for network in networks:
---
-2.20.1
-
-From 627bf1daaae1510cfd4016297ed16b82df209aae Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 18 Nov 2020 13:33:45 +0000
-Subject: [PATCH 071/111] python: Remove unnecessary db object from writers
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/export.py | 12 ++++++------
- 1 file changed, 6 insertions(+), 6 deletions(-)
-
-diff --git a/src/python/export.py b/src/python/export.py
-index 4219957..5bc9f30 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -39,8 +39,8 @@ class OutputWriter(object):
- suffix = "networks"
- mode = "w"
+-LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
++size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
+ size_t counter = 0;
-- def __init__(self, db, f, prefix=None, flatten=True):
-- self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten
-+ def __init__(self, f, prefix=None, flatten=True):
-+ self.f, self.prefix, self.flatten = f, prefix, flatten
+ int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
+@@ -661,11 +1000,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
+ return counter;
+ }
- # The previously written network
- self._last_network = None
-@@ -49,13 +49,13 @@ class OutputWriter(object):
- self._write_header()
+-LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
++size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
+ return __loc_network_tree_count_nodes(tree->root);
+ }
- @classmethod
-- def open(cls, db, filename, **kwargs):
-+ def open(cls, filename, **kwargs):
- """
- Convenience function to open a file
- """
- f = open(filename, cls.mode)
+-LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
++int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
+ struct loc_network_tree_node* n = calloc(1, sizeof(*n));
+ if (!n)
+ return -ENOMEM;
+@@ -680,7 +1019,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
+ return 0;
+ }
-- return cls(db, f, **kwargs)
-+ return cls(f, **kwargs)
+-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
++struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
+ if (node)
+ node->refcount++;
- def __repr__(self):
- return "<%s f=%s>" % (self.__class__.__name__, self.f)
-@@ -172,7 +172,7 @@ class Exporter(object):
- directory, prefix=country_code, suffix=self.writer.suffix, family=family,
- )
+@@ -703,7 +1042,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
+ free(node);
+ }
-- writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code)
-+ writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
+-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
++struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
+ if (!node)
+ return NULL;
- # Create writers for ASNs
- for asn in asns:
-@@ -180,7 +180,7 @@ class Exporter(object):
- directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
- )
+@@ -714,7 +1053,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
+ return NULL;
+ }
-- writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
-+ writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
-
- # Filter countries from special country codes
- country_codes = [
---
-2.20.1
-
-From 9cb56ac9adafafa6e452009c2fa2d42e94474e11 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 18 Nov 2020 13:34:50 +0000
-Subject: [PATCH 072/111] location: End lookup after an invalid IP address was
- passed
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/location.in | 1 +
- 1 file changed, 1 insertion(+)
-
-diff --git a/src/python/location.in b/src/python/location.in
-index 0d09210..6885ea0 100644
---- a/src/python/location.in
-+++ b/src/python/location.in
-@@ -253,6 +253,7 @@ class CLI(object):
- network = db.lookup(address)
- except ValueError:
- print(_("Invalid IP address: %s") % address, file=sys.stderr)
-+ return 2
+-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
++struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
+ if (index == 0)
+ node = node->zero;
+ else
+@@ -726,10 +1065,10 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
+ return loc_network_tree_node_ref(node);
+ }
- args = {
- "address" : address,
---
-2.20.1
-
-From 2a550d12208f8bc8002e05ac08613312df26b20e Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 19 Nov 2020 12:03:33 +0000
-Subject: [PATCH 073/111] python: Fix download of database
-
-This was all messed up in 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1
-when the change of type for the timestamp wasn't changed everywhere
-else.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/downloader.py | 6 +++---
- src/python/location.in | 2 +-
- 2 files changed, 4 insertions(+), 4 deletions(-)
-
-diff --git a/src/python/downloader.py b/src/python/downloader.py
-index 87bbb68..05f7872 100644
---- a/src/python/downloader.py
-+++ b/src/python/downloader.py
-@@ -119,8 +119,8 @@ class Downloader(object):
+-LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
++int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
+ return (!!node->network);
+ }
- headers = {}
- if timestamp:
-- headers["If-Modified-Since"] = timestamp.strftime(
-- "%a, %d %b %Y %H:%M:%S GMT",
-+ headers["If-Modified-Since"] = time.strftime(
-+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
- )
+-LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
++struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
+ return loc_network_ref(node->network);
+ }
+diff --git a/src/perl/Location.xs b/src/perl/Location.xs
+index dcf3f0d..b7676d2 100644
+--- a/src/perl/Location.xs
++++ b/src/perl/Location.xs
+@@ -125,7 +125,7 @@ database_countries(db)
+ PPCODE:
+ // Create Database enumerator
+ struct loc_database_enumerator* enumerator;
+- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
++ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
- t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
-@@ -195,7 +195,7 @@ class Downloader(object):
- db = Database(f.name)
+ if (err) {
+ croak("Could not create a database enumerator\n");
+diff --git a/src/python/database.c b/src/python/database.c
+index 1013a58..f385c61 100644
+--- a/src/python/database.c
++++ b/src/python/database.c
+@@ -17,6 +17,8 @@
+ #include <Python.h>
- # Database is not recent
-- if timestamp and db.created_at < timestamp.timestamp():
-+ if timestamp and db.created_at < timestamp:
- return False
+ #include <loc/libloc.h>
++#include <loc/as.h>
++#include <loc/as-list.h>
+ #include <loc/database.h>
- log.info("Downloaded new database from %s" % (time.strftime(
-diff --git a/src/python/location.in b/src/python/location.in
-index 6885ea0..b30beae 100644
---- a/src/python/location.in
-+++ b/src/python/location.in
-@@ -433,7 +433,7 @@ class CLI(object):
+ #include "locationmodule.h"
+@@ -207,10 +209,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
+ return (PyObject*)self;
+ }
- # Try downloading a new database
- try:
-- t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
-+ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
+-static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
++static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
+ struct loc_database_enumerator* enumerator;
- # If no file could be downloaded, log a message
- except FileNotFoundError as e:
---
-2.20.1
-
-From a1a00053300cff3c0f690d52377c76c83c2a08b2 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 19 Nov 2020 12:34:11 +0000
-Subject: [PATCH 074/111] python: Add property to return IP addresses as bytes
-
-This avoids calling inet_pton to parse IP addresses from string
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 2 ++
- src/loc/network.h | 2 ++
- src/network.c | 8 ++++++++
- src/python/database.c | 2 +-
- src/python/export.py | 7 ++-----
- src/python/network.c | 40 ++++++++++++++++++++++++++++++++++++++++
- 6 files changed, 55 insertions(+), 6 deletions(-)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index 53273cd..406dd15 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -113,6 +113,8 @@ global:
- loc_network_format_last_address;
- loc_network_get_asn;
- loc_network_get_country_code;
-+ loc_network_get_first_address;
-+ loc_network_get_last_address;
- loc_network_gt;
- loc_network_has_flag;
- loc_network_is_subnet;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index d86b685..4b7410c 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -39,7 +39,9 @@ struct loc_network* loc_network_unref(struct loc_network* network);
- char* loc_network_str(struct loc_network* network);
- int loc_network_address_family(struct loc_network* network);
+- int r = loc_database_enumerator_new(&enumerator, self->db, what);
++ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
+ if (r) {
+ PyErr_SetFromErrno(PyExc_SystemError);
+ return NULL;
+@@ -223,7 +225,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
+ }
-+const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
- char* loc_network_format_first_address(struct loc_network* network);
-+const struct in6_addr* loc_network_get_last_address(struct loc_network* network);
- char* loc_network_format_last_address(struct loc_network* network);
- int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
+ static PyObject* Database_ases(DatabaseObject* self) {
+- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
++ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
+ }
-diff --git a/src/network.c b/src/network.c
-index 28ca2df..4c8787a 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -343,10 +343,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc
- return string;
+ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
+@@ -234,7 +236,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
+
+ struct loc_database_enumerator* enumerator;
+
+- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
+ if (r) {
+ PyErr_SetFromErrno(PyExc_SystemError);
+ return NULL;
+@@ -250,44 +252,142 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
}
-+LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) {
-+ return &network->first_address;
+ static PyObject* Database_networks(DatabaseObject* self) {
+- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
+}
+
- LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) {
- return loc_network_format_address(network, &network->first_address);
- }
-
-+LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) {
-+ return &network->last_address;
-+}
++static PyObject* Database_networks_flattened(DatabaseObject *self) {
++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
+ }
+
+ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
+- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
+- const char* country_code = NULL;
+- unsigned int asn = 0;
++ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
++ PyObject* country_codes = NULL;
++ PyObject* asn_list = NULL;
+ int flags = 0;
+ int family = 0;
++ int flatten = 0;
+
+- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
++ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
+ return NULL;
+
+ struct loc_database_enumerator* enumerator;
+- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
++ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
+ if (r) {
+ PyErr_SetFromErrno(PyExc_SystemError);
+ return NULL;
+ }
+
+ // Set country code we are searching for
+- if (country_code) {
+- r = loc_database_enumerator_set_country_code(enumerator, country_code);
++ if (country_codes) {
++ struct loc_country_list* countries;
++ r = loc_country_list_new(loc_ctx, &countries);
++ if (r) {
++ PyErr_SetString(PyExc_SystemError, "Could not create country list");
++ return NULL;
++ }
++
++ for (unsigned int i = 0; i < PyList_Size(country_codes); i++) {
++ PyObject* item = PyList_GetItem(country_codes, i);
++
++ if (!PyUnicode_Check(item)) {
++ PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
++ loc_country_list_unref(countries);
++ return NULL;
++ }
++
++ const char* country_code = PyUnicode_AsUTF8(item);
++
++ struct loc_country* country;
++ r = loc_country_new(loc_ctx, &country, country_code);
++ if (r) {
++ if (r == -EINVAL) {
++ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
++ } else {
++ PyErr_SetString(PyExc_SystemError, "Could not create country");
++ }
++
++ loc_country_list_unref(countries);
++ return NULL;
++ }
++
++ // Append it to the list
++ r = loc_country_list_append(countries, country);
++ if (r) {
++ PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
+
- LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
- return loc_network_format_address(network, &network->last_address);
- }
-diff --git a/src/python/database.c b/src/python/database.c
-index ed22275..f385c61 100644
---- a/src/python/database.c
-+++ b/src/python/database.c
-@@ -329,7 +329,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
++ loc_country_list_unref(countries);
++ loc_country_unref(country);
++ return NULL;
++ }
++
++ loc_country_unref(country);
++ }
+
++ r = loc_database_enumerator_set_countries(enumerator, countries);
if (r) {
PyErr_SetFromErrno(PyExc_SystemError);
-
-- loc_as_list_unref(countries);
++
+ loc_country_list_unref(countries);
return NULL;
}
++
++ loc_country_list_unref(countries);
+ }
-diff --git a/src/python/export.py b/src/python/export.py
-index 5bc9f30..6b39878 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -142,11 +142,8 @@ class XTGeoIPOutputWriter(OutputWriter):
- mode = "wb"
-
- def _write_network(self, network):
-- for address in (network.first_address, network.last_address):
-- # Convert this into a string of bits
-- bytes = socket.inet_pton(network.family, address)
--
-- self.f.write(bytes)
-+ for address in (network._first_address, network._last_address):
-+ self.f.write(address)
-
-
- formats = {
-diff --git a/src/python/network.c b/src/python/network.c
-index ed91d65..742b472 100644
---- a/src/python/network.c
-+++ b/src/python/network.c
-@@ -215,6 +215,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) {
- return obj;
- }
+ // Set the ASN we are searching for
+- if (asn) {
+- r = loc_database_enumerator_set_asn(enumerator, asn);
++ if (asn_list) {
++ struct loc_as_list* asns;
++ r = loc_as_list_new(loc_ctx, &asns);
++ if (r) {
++ PyErr_SetString(PyExc_SystemError, "Could not create AS list");
++ return NULL;
++ }
++
++ for (unsigned int i = 0; i < PyList_Size(asn_list); i++) {
++ PyObject* item = PyList_GetItem(asn_list, i);
++
++ if (!PyLong_Check(item)) {
++ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
-+static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) {
-+ struct in_addr address4;
++ loc_as_list_unref(asns);
++ return NULL;
++ }
+
-+ // Convert IPv4 addresses to struct in_addr
-+ if (IN6_IS_ADDR_V4MAPPED(address6)) {
-+ address4.s_addr = address6->s6_addr32[3];
++ unsigned long number = PyLong_AsLong(item);
+
-+ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4));
-+ }
++ struct loc_as* as;
++ r = loc_as_new(loc_ctx, &as, number);
++ if (r) {
++ PyErr_SetString(PyExc_SystemError, "Could not create AS");
+
-+ // Return IPv6 addresses as they are
-+ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6));
-+}
++ loc_as_list_unref(asns);
++ loc_as_unref(as);
++ return NULL;
++ }
+
-+static PyObject* Network_get__first_address(NetworkObject* self) {
-+ const struct in6_addr* address = loc_network_get_first_address(self->network);
++ r = loc_as_list_append(asns, as);
++ if (r) {
++ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
+
-+ return PyBytes_FromAddress(address);
-+}
++ loc_as_list_unref(asns);
++ loc_as_unref(as);
++ return NULL;
++ }
+
- static PyObject* Network_get_last_address(NetworkObject* self) {
- char* address = loc_network_format_last_address(self->network);
++ loc_as_unref(as);
++ }
++
++ r = loc_database_enumerator_set_asns(enumerator, asns);
+ if (r) {
+ PyErr_SetFromErrno(PyExc_SystemError);
++
++ loc_as_list_unref(asns);
+ return NULL;
+ }
++
++ loc_as_list_unref(asns);
+ }
-@@ -224,6 +244,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
- return obj;
+ // Set the flags we are searching for
+@@ -317,7 +417,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
}
-+static PyObject* Network_get__last_address(NetworkObject* self) {
-+ const struct in6_addr* address = loc_network_get_last_address(self->network);
-+
-+ return PyBytes_FromAddress(address);
-+}
-+
- static struct PyMethodDef Network_methods[] = {
- {
- "exclude",
-@@ -281,6 +307,13 @@ static struct PyGetSetDef Network_getsetters[] = {
+ static PyObject* Database_countries(DatabaseObject* self) {
+- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
++ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
+ }
+
+ static struct PyMethodDef Database_methods[] = {
+@@ -403,6 +503,13 @@ static struct PyGetSetDef Database_getsetters[] = {
NULL,
NULL,
},
+ {
-+ "_first_address",
-+ (getter)Network_get__first_address,
++ "networks_flattened",
++ (getter)Database_networks_flattened,
+ NULL,
+ NULL,
+ NULL,
+ },
{
- "last_address",
- (getter)Network_get_last_address,
-@@ -288,6 +321,13 @@ static struct PyGetSetDef Network_getsetters[] = {
- NULL,
- NULL,
- },
-+ {
-+ "_last_address",
-+ (getter)Network_get__last_address,
-+ NULL,
-+ NULL,
-+ NULL,
-+ },
- { NULL },
- };
+ "vendor",
+ (getter)Database_get_vendor,
+diff --git a/src/python/downloader.py b/src/python/downloader.py
+index 87bbb68..05f7872 100644
+--- a/src/python/downloader.py
++++ b/src/python/downloader.py
+@@ -119,8 +119,8 @@ class Downloader(object):
+
+ headers = {}
+ if timestamp:
+- headers["If-Modified-Since"] = timestamp.strftime(
+- "%a, %d %b %Y %H:%M:%S GMT",
++ headers["If-Modified-Since"] = time.strftime(
++ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
+ )
+
+ t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
+@@ -195,7 +195,7 @@ class Downloader(object):
+ db = Database(f.name)
+
+ # Database is not recent
+- if timestamp and db.created_at < timestamp.timestamp():
++ if timestamp and db.created_at < timestamp:
+ return False
---
-2.20.1
-
-From 90d2194a876c223f9124ce9e27bdee6a6b49ff6a Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 19 Nov 2020 12:40:01 +0000
-Subject: [PATCH 075/111] export: Remove old flattening feature
-
-The database enumerator now only returns networks that will
-never overlap.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/export.py | 34 ++++++----------------------------
- 1 file changed, 6 insertions(+), 28 deletions(-)
-
+ log.info("Downloaded new database from %s" % (time.strftime(
diff --git a/src/python/export.py b/src/python/export.py
-index 6b39878..4702bcf 100644
+index d15c6f0..f0eae26 100644
--- a/src/python/export.py
+++ b/src/python/export.py
+@@ -29,7 +29,7 @@ import _location
+ log = logging.getLogger("location.export")
+ log.propagate = 1
+
+-flags = {
++FLAGS = {
+ _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
+ _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
+ _location.NETWORK_FLAG_ANYCAST : "A3",
@@ -39,11 +39,8 @@ class OutputWriter(object):
suffix = "networks"
mode = "w"
def _write_header(self):
"""
The header of the file
-@@ -84,15 +69,8 @@ class OutputWriter(object):
+@@ -84,16 +69,8 @@ class OutputWriter(object):
"""
pass
-
def write(self, network):
- if self.flatten and self._flatten(network):
-- log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
+- log.debug("Skipping writing network %s" % network)
- return
-
-- return self._write_network(network)
+- # Write the network to file
+- self._write_network(network)
+ self.f.write("%s\n" % network)
def finish(self):
"""
-@@ -113,7 +91,7 @@ class IpsetOutputWriter(OutputWriter):
+@@ -114,7 +91,7 @@ class IpsetOutputWriter(OutputWriter):
def _write_header(self):
self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
self.f.write("add %s %s\n" % (self.prefix, network))
-@@ -129,7 +107,7 @@ class NftablesOutputWriter(OutputWriter):
+@@ -130,7 +107,7 @@ class NftablesOutputWriter(OutputWriter):
def _write_footer(self):
self.f.write("}\n")
- def _write_network(self, network):
+ def write(self, network):
- self.f.write(" %s,\n" % network)
-
-
-@@ -141,7 +119,7 @@ class XTGeoIPOutputWriter(OutputWriter):
- suffix = "iv"
- mode = "wb"
-
-- def _write_network(self, network):
-+ def write(self, network):
- for address in (network._first_address, network._last_address):
- self.f.write(address)
-
---
-2.20.1
-
-From 90188dad86223380b0854b523b63ec024117c5f2 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 19 Nov 2020 12:41:19 +0000
-Subject: [PATCH 076/111] export: Speed-up export in xt_geoip format
-
-Removing the loop avoids creating a tuple and then iterating over it
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/python/export.py | 4 ++--
- 1 file changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/src/python/export.py b/src/python/export.py
-index 4702bcf..f0eae26 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -120,8 +120,8 @@ class XTGeoIPOutputWriter(OutputWriter):
- mode = "wb"
-
- def write(self, network):
-- for address in (network._first_address, network._last_address):
-- self.f.write(address)
-+ self.f.write(network._first_address)
-+ self.f.write(network._last_address)
-
-
- formats = {
---
-2.20.1
-
-From 6661692f3bc8f788af8b75ae25561f4cc4a2acb5 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 19 Nov 2020 12:48:46 +0000
-Subject: [PATCH 077/111] database: Disable some useless code when not running
- in debug mode
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 16 +++++++++++++++-
- 1 file changed, 15 insertions(+), 1 deletion(-)
-
-diff --git a/src/database.c b/src/database.c
-index 83dd752..ef4f505 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -619,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
- }
-
- clock_t end = clock();
-- DEBUG(db->ctx, "Signature checked in %.4fms\n",
-+ INFO(db->ctx, "Signature checked in %.4fms\n",
- (double)(end - start) / CLOCKS_PER_SEC * 1000);
-
- CLEANUP:
-@@ -679,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
- off_t lo = 0;
- off_t hi = db->as_count - 1;
-
-+#ifdef ENABLE_DEBUG
- // Save start time
- clock_t start = clock();
-+#endif
-
- while (lo <= hi) {
- off_t i = (lo + hi) / 2;
-@@ -693,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
- // Check if this is a match
- uint32_t as_number = loc_as_get_number(*as);
- if (as_number == number) {
-+#ifdef ENABLE_DEBUG
- clock_t end = clock();
-
- // Log how fast this has been
- DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
- (double)(end - start) / CLOCKS_PER_SEC * 1000);
-+#endif
-
- return 0;
- }
-@@ -741,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
- return -1;
- }
-
-+#ifdef ENABLE_DEBUG
- if (r == 0) {
- char* string = loc_network_str(*network);
- DEBUG(db->ctx, "Got network %s\n", string);
- free(string);
- }
-+#endif
-
- return r;
- }
-@@ -840,17 +846,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
-
- *network = NULL;
-
-+#ifdef ENABLE_DEBUG
- // Save start time
- clock_t start = clock();
-+#endif
+ self.f.write(" %s,\n" % network)
- int r = __loc_database_lookup(db, address, network, &network_address,
- db->network_nodes_v1, 0);
-+#ifdef ENABLE_DEBUG
- clock_t end = clock();
+@@ -142,14 +119,9 @@ class XTGeoIPOutputWriter(OutputWriter):
+ suffix = "iv"
+ mode = "wb"
- // Log how fast this has been
- DEBUG(db->ctx, "Executed network search in %.4fms\n",
- (double)(end - start) / CLOCKS_PER_SEC * 1000);
-+#endif
+- def _write_network(self, network):
+- for address in (network.first_address, network.last_address):
+- # Convert this into a string of bits
+- bytes = socket.inet_pton(
+- network.family, address,
+- )
+-
+- self.f.write(bytes)
++ def write(self, network):
++ self.f.write(network._first_address)
++ self.f.write(network._last_address)
- return r;
- }
-@@ -897,8 +907,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
- off_t lo = 0;
- off_t hi = db->countries_count - 1;
-+#ifdef ENABLE_DEBUG
- // Save start time
- clock_t start = clock();
-+#endif
+ formats = {
+@@ -185,8 +157,14 @@ class Exporter(object):
- while (lo <= hi) {
- off_t i = (lo + hi) / 2;
-@@ -913,11 +925,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
- int result = strcmp(code, cc);
+ writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
- if (result == 0) {
-+#ifdef ENABLE_DEBUG
- clock_t end = clock();
++ # Filter countries from special country codes
++ country_codes = [
++ country_code for country_code in countries if not country_code in FLAGS.values()
++ ]
++
+ # Get all networks that match the family
+- networks = self.db.search_networks(family=family)
++ networks = self.db.search_networks(family=family,
++ country_codes=country_codes, asns=asns, flatten=True)
- // Log how fast this has been
- DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
- (double)(end - start) / CLOCKS_PER_SEC * 1000);
-+#endif
+ # Walk through all networks
+ for network in networks:
+@@ -203,10 +181,10 @@ class Exporter(object):
+ pass
- return 0;
- }
---
-2.20.1
-
-From a17c353bce55a9f38f67b4b7d2425194cfa208e7 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 20 Nov 2020 14:43:31 +0000
-Subject: [PATCH 078/111] test: Add tests to network-lists
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- Makefile.am | 10 ++++
- src/.gitignore | 1 +
- src/test-network-list.c | 126 ++++++++++++++++++++++++++++++++++++++++
- 3 files changed, 137 insertions(+)
- create mode 100644 src/test-network-list.c
-
-diff --git a/Makefile.am b/Makefile.am
-index d0cc793..ebd7e17 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -318,6 +318,7 @@ check_PROGRAMS = \
- src/test-database \
- src/test-as \
- src/test-network \
-+ src/test-network-list \
- src/test-country \
- src/test-signature
+ # Handle flags
+- for flag in flags:
++ for flag in FLAGS:
+ if network.has_flag(flag):
+ # Fetch the "fake" country code
+- country = flags[flag]
++ country = FLAGS[flag]
-@@ -357,6 +358,15 @@ src_test_network_CFLAGS = \
- src_test_network_LDADD = \
- src/libloc.la
+ try:
+ writers[country].write(network)
+diff --git a/src/python/importer.py b/src/python/importer.py
+index f19db4b..5f46bc3 100644
+--- a/src/python/importer.py
++++ b/src/python/importer.py
+@@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
+ "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
-+src_test_network_list_SOURCES = \
-+ src/test-network-list.c
-+
-+src_test_network_list_CFLAGS = \
-+ $(TESTS_CFLAGS)
-+
-+src_test_network_list_LDADD = \
-+ src/libloc.la
-+
- src_test_stringpool_SOURCES = \
- src/test-stringpool.c
+ # Latin America and Caribbean Network Information Centre
+- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
++ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
-diff --git a/src/.gitignore b/src/.gitignore
-index caf80b5..3ccbdb8 100644
---- a/src/.gitignore
-+++ b/src/.gitignore
-@@ -10,5 +10,6 @@ test-libloc
- test-database
- test-country
- test-network
-+test-network-list
- test-signature
- test-stringpool
-diff --git a/src/test-network-list.c b/src/test-network-list.c
-new file mode 100644
-index 0000000..3061d63
---- /dev/null
-+++ b/src/test-network-list.c
-@@ -0,0 +1,126 @@
-+/*
-+ libloc - A library to determine the location of someone on the Internet
-+
-+ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
-+
-+ This program is free software; you can redistribute it and/or modify
-+ it under the terms of the GNU General Public License as published by
-+ the Free Software Foundation; either version 2 of the License, or
-+ (at your option) any later version.
-+
-+ This program is distributed in the hope that it will be useful,
-+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-+ GNU General Public License for more details.
-+*/
-+
-+#include <errno.h>
-+#include <stdio.h>
-+#include <stddef.h>
-+#include <stdlib.h>
-+#include <string.h>
-+#include <syslog.h>
-+
-+#include <loc/libloc.h>
-+#include <loc/network.h>
-+#include <loc/network-list.h>
-+
-+int main(int argc, char** argv) {
-+ int err;
-+
-+ struct loc_ctx* ctx;
-+ err = loc_new(&ctx);
-+ if (err < 0)
-+ exit(EXIT_FAILURE);
-+
-+ // Enable debug logging
-+ loc_set_log_priority(ctx, LOG_DEBUG);
-+
-+ // Create a network
-+ struct loc_network* network1;
-+ err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32");
-+ if (err) {
-+ fprintf(stderr, "Could not create the network1\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ struct loc_network* subnet1;
-+ err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48");
-+ if (err) {
-+ fprintf(stderr, "Could not create the subnet1\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ struct loc_network* subnet2;
-+ err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48");
-+ if (err) {
-+ fprintf(stderr, "Could not create the subnet2\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Make a list with both subnets
-+ struct loc_network_list* subnets;
-+ err = loc_network_list_new(ctx, &subnets);
-+ if (err) {
-+ fprintf(stderr, "Could not create subnets list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ size_t size = loc_network_list_size(subnets);
-+ if (size > 0) {
-+ fprintf(stderr, "The list is not empty: %zu\n", size);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ err = loc_network_list_push(subnets, subnet1);
-+ if (err) {
-+ fprintf(stderr, "Could not add subnet1 to subnets list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ if (loc_network_list_empty(subnets)) {
-+ fprintf(stderr, "The subnets list reports that it is empty\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ err = loc_network_list_push(subnets, subnet2);
-+ if (err) {
-+ fprintf(stderr, "Could not add subnet2 to subnets list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ size = loc_network_list_size(subnets);
-+ if (size != 2) {
-+ fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Exclude subnet1 from network1
-+ struct loc_network_list* excluded = loc_network_exclude(network1, subnet1);
-+ if (!excluded) {
-+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ loc_network_list_dump(excluded);
-+
-+ // Exclude all subnets from network1
-+ excluded = loc_network_exclude_list(network1, subnets);
-+ if (!excluded) {
-+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ loc_network_list_dump(excluded);
-+
-+ if (excluded)
-+ loc_network_list_unref(excluded);
-+
-+ loc_network_list_unref(subnets);
-+ loc_network_unref(network1);
-+ loc_network_unref(subnet1);
-+ loc_network_unref(subnet2);
-+ loc_unref(ctx);
-+
-+ return EXIT_SUCCESS;
-+}
---
-2.20.1
-
-From 1209ff0cd4be6b2f52a5e82168db8a8ac68e628a Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 20 Nov 2020 16:24:40 +0000
-Subject: [PATCH 079/111] network: Fix loc_network_is_subnet()
-
-This function always returned false.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 16 +++-------------
- 1 file changed, 3 insertions(+), 13 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index 4c8787a..b440d76 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -491,12 +491,12 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
- LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
- // If the start address of the other network is smaller than this network,
- // it cannot be a subnet.
-- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
-+ if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
- return 0;
+ # Réseaux IP Européens
+ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+diff --git a/src/python/location-importer.in b/src/python/location-importer.in
+index 1467923..2dec89e 100644
+--- a/src/python/location-importer.in
++++ b/src/python/location-importer.in
+@@ -152,6 +152,7 @@ class CLI(object):
+ last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
+ CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
+ CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
++ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
+
+ -- autnums
+ CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
+@@ -165,6 +166,7 @@ class CLI(object):
+ -- networks
+ CREATE TABLE IF NOT EXISTS networks(network inet, country text);
+ CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
++ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
+ CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
- // If the end address of the other network is greater than this network,
- // it cannot be a subnet.
-- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
-+ if (in6_addr_cmp(&self->last_address, &other->last_address) < 0)
- return 0;
+ -- overrides
+@@ -188,6 +190,8 @@ class CLI(object):
+ );
+ CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
+ ON network_overrides(network);
++ CREATE INDEX IF NOT EXISTS network_overrides_search
++ ON network_overrides USING GIST(network inet_ops);
+ """)
- return 1;
-@@ -504,17 +504,7 @@ LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_networ
+ return db
+@@ -234,32 +238,24 @@ class CLI(object):
- // XXX DEPRECATED - I find this too difficult to use
- LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
-- // If the start address of the other network is smaller than this network,
-- // it cannot be a subnet.
-- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
-- return 0;
--
-- // If the end address of the other network is greater than this network,
-- // it cannot be a subnet.
-- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
-- return 0;
+ # Select all known networks
+ rows = self.db.query("""
+- -- Get a (sorted) list of all known networks
+- WITH known_networks AS (
+- SELECT network FROM announcements
+- UNION
+- SELECT network FROM networks
+- ORDER BY network
+- )
-
-- return 1;
-+ return loc_network_is_subnet(other, self);
- }
-
- LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
---
-2.20.1
-
-From 92349c14876f8e4711739c82d1a389a53bcb27a5 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 20 Nov 2020 16:25:27 +0000
-Subject: [PATCH 080/111] network: Remove debugging output
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 4 ----
- 1 file changed, 4 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index b440d76..3271272 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -670,10 +670,6 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude(
- loc_network_unref(subnet2);
- }
+ -- Return a list of those networks enriched with all
+ -- other information that we store in the database
+ SELECT
+- DISTINCT ON (known_networks.network)
+- known_networks.network AS network,
+- announcements.autnum AS autnum,
++ DISTINCT ON (network)
++ network,
++ autnum,
--#ifdef ENABLE_DEBUG
-- loc_network_list_dump(list);
--#endif
+ -- Country
+ COALESCE(
+ (
+ SELECT country FROM network_overrides overrides
+- WHERE announcements.network <<= overrides.network
++ WHERE networks.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT country FROM autnum_overrides overrides
+- WHERE announcements.autnum = overrides.number
++ WHERE networks.autnum = overrides.number
+ ),
+ networks.country
+ ) AS country,
+@@ -268,50 +264,67 @@ class CLI(object):
+ COALESCE(
+ (
+ SELECT is_anonymous_proxy FROM network_overrides overrides
+- WHERE announcements.network <<= overrides.network
++ WHERE networks.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT is_anonymous_proxy FROM autnum_overrides overrides
+- WHERE announcements.autnum = overrides.number
++ WHERE networks.autnum = overrides.number
+ ),
+ FALSE
+ ) AS is_anonymous_proxy,
+ COALESCE(
+ (
+ SELECT is_satellite_provider FROM network_overrides overrides
+- WHERE announcements.network <<= overrides.network
++ WHERE networks.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT is_satellite_provider FROM autnum_overrides overrides
+- WHERE announcements.autnum = overrides.number
++ WHERE networks.autnum = overrides.number
+ ),
+ FALSE
+ ) AS is_satellite_provider,
+ COALESCE(
+ (
+ SELECT is_anycast FROM network_overrides overrides
+- WHERE announcements.network <<= overrides.network
++ WHERE networks.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT is_anycast FROM autnum_overrides overrides
+- WHERE announcements.autnum = overrides.number
++ WHERE networks.autnum = overrides.number
+ ),
+ FALSE
+- ) AS is_anycast,
-
- // Return the result
- return list;
-
---
-2.20.1
-
-From ed0f53df0f3ebb915faf25138cc09df7555415a3 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 20 Nov 2020 16:25:56 +0000
-Subject: [PATCH 081/111] database: Flatten out code due to compiler errors
-
-It looks like GCC could not follow the code as it was
-written. Therefore I clean up more often now instead
-of having a GOTO block where I do that in.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 59 ++++++++++++++++++++++++++++++++++----------------
- 1 file changed, 40 insertions(+), 19 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index ef4f505..c21b957 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1305,8 +1305,12 @@ static int __loc_database_enumerator_next_network_flattened(
- while (1) {
- // Fetch the next network in line
- r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
-- if (r)
-- goto END;
-+ if (r) {
-+ loc_network_unref(subnet);
-+ loc_network_list_unref(subnets);
+- -- Must be part of returned values for ORDER BY clause
+- masklen(announcements.network) AS sort_a,
+- masklen(networks.network) AS sort_b
+- FROM known_networks
+- LEFT JOIN announcements ON known_networks.network <<= announcements.network
+- LEFT JOIN networks ON known_networks.network <<= networks.network
+- ORDER BY known_networks.network, sort_a DESC, sort_b DESC
++ ) AS is_anycast
++ FROM (
++ SELECT
++ known_networks.network AS network,
++ announcements.autnum AS autnum,
++ networks.country AS country,
+
-+ return r;
-+ }
++ -- Must be part of returned values for ORDER BY clause
++ masklen(announcements.network) AS sort_a,
++ masklen(networks.network) AS sort_b
++ FROM (
++ SELECT network FROM announcements
++ UNION ALL
++ SELECT network FROM networks
++ UNION ALL
++ SELECT network FROM network_overrides
++ ) known_networks
++ LEFT JOIN
++ announcements ON known_networks.network <<= announcements.network
++ LEFT JOIN
++ networks ON known_networks.network <<= networks.network
++ ORDER BY
++ known_networks.network,
++ sort_a DESC,
++ sort_b DESC
++ ) networks
+ """)
- // End if we did not receive another subnet
- if (!subnet)
-@@ -1315,8 +1319,12 @@ static int __loc_database_enumerator_next_network_flattened(
- // Collect all subnets in a list
- if (loc_network_is_subnet(*network, subnet)) {
- r = loc_network_list_push(subnets, subnet);
-- if (r)
-- goto END;
-+ if (r) {
-+ loc_network_unref(subnet);
-+ loc_network_list_unref(subnets);
+ for row in rows:
+@@ -363,6 +376,16 @@ class CLI(object):
+ CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
+ ON COMMIT DROP;
+ CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
+
-+ return r;
-+ }
++ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
++ ON COMMIT DROP;
++ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
++ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
++ """)
++
++ # Remove all previously imported content
++ self.db.execute("""
++ TRUNCATE TABLE networks;
+ """)
- loc_network_unref(subnet);
- continue;
-@@ -1324,8 +1332,12 @@ static int __loc_database_enumerator_next_network_flattened(
+ for source in location.importer.WHOIS_SOURCES:
+@@ -370,31 +393,72 @@ class CLI(object):
+ for block in f:
+ self._parse_block(block)
- // If this is not a subnet, we push it back onto the stack and break
- r = loc_network_list_push(enumerator->stack, subnet);
-- if (r)
-- goto END;
-+ if (r) {
-+ loc_network_unref(subnet);
-+ loc_network_list_unref(subnets);
++ # Process all parsed networks from every RIR we happen to have access to,
++ # insert the largest network chunks into the networks table immediately...
++ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
+
-+ return r;
-+ }
-
- loc_network_unref(subnet);
- break;
-@@ -1343,29 +1355,38 @@ static int __loc_database_enumerator_next_network_flattened(
- // If the network has any subnets, we will break it into smaller parts
- // without the subnets.
- struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
-- if (!excluded || loc_network_list_empty(excluded)) {
-- r = 1;
-- goto END;
-+ if (!excluded) {
-+ loc_network_list_unref(subnets);
-+ return -1;
-+ }
++ for family in (row.family for row in families):
++ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
+
-+ // Merge excluded list with subnets
-+ r = loc_network_list_merge(subnets, excluded);
-+ if (r) {
-+ loc_network_list_unref(subnets);
-+ loc_network_list_unref(excluded);
++ self.db.execute("INSERT INTO networks(network, country) \
++ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
+
-+ return r;
- }
-
-+ // We no longer need the excluded list
-+ loc_network_list_unref(excluded);
++ # ... determine any other prefixes for this network family, ...
++ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
++ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
+
-+ // Sort all subnets
-+ loc_network_list_sort(subnets);
++ # ... and insert networks with this prefix in case they provide additional
++ # information (i. e. subnet of a larger chunk with a different country)
++ for prefix in (row.prefix for row in prefixes):
++ self.db.execute("""
++ WITH candidates AS (
++ SELECT
++ _rirdata.network,
++ _rirdata.country
++ FROM
++ _rirdata
++ WHERE
++ family(_rirdata.network) = %s
++ AND
++ masklen(_rirdata.network) = %s
++ ),
++ filtered AS (
++ SELECT
++ DISTINCT ON (c.network)
++ c.network,
++ c.country,
++ masklen(networks.network),
++ networks.country AS parent_country
++ FROM
++ candidates c
++ LEFT JOIN
++ networks
++ ON
++ c.network << networks.network
++ ORDER BY
++ c.network,
++ masklen(networks.network) DESC NULLS LAST
++ )
++ INSERT INTO
++ networks(network, country)
++ SELECT
++ network,
++ country
++ FROM
++ filtered
++ WHERE
++ parent_country IS NULL
++ OR
++ country <> parent_country
++ ON CONFLICT DO NOTHING""",
++ family, prefix,
++ )
+
- // Replace network with the first one
- loc_network_unref(*network);
-
-- *network = loc_network_list_pop_first(excluded);
-+ *network = loc_network_list_pop_first(subnets);
-
- // Push the rest onto the stack
-- loc_network_list_reverse(excluded);
-- loc_network_list_merge(enumerator->stack, excluded);
+ self.db.execute("""
+ INSERT INTO autnums(number, name)
+ SELECT _autnums.number, _organizations.name FROM _autnums
+ JOIN _organizations ON _autnums.organization = _organizations.handle
+- ON CONFLICT (number) DO UPDATE SET name = excluded.name
+- """)
+-
+- self.db.execute("""
+- --- Purge any redundant entries
+- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
+- AS
+- SELECT network FROM networks candidates
+- WHERE EXISTS (
+- SELECT FROM networks
+- WHERE
+- networks.network << candidates.network
+- AND
+- networks.country = candidates.country
+- );
-
-- loc_network_list_unref(excluded);
+- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
-
--END:
-- if (subnet)
-- loc_network_unref(subnet);
-+ loc_network_list_reverse(subnets);
-+ loc_network_list_merge(enumerator->stack, subnets);
-
- loc_network_list_unref(subnets);
-
-- return r;
-+ return 0;
- }
-
- LOC_EXPORT int loc_database_enumerator_next_network(
---
-2.20.1
-
-From a1024390795d60fab9f697fa230c9901ebd7d221 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 20 Nov 2020 16:42:55 +0000
-Subject: [PATCH 082/111] network-list: Implement merging in reverse in one
- step
-
-This will save us some time because we do not need to
-change the list in place first and then merge it.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 3 +--
- src/libloc.sym | 1 +
- src/loc/network-list.h | 1 +
- src/network-list.c | 13 +++++++++++++
- 4 files changed, 16 insertions(+), 2 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index c21b957..ba4f98a 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1381,8 +1381,7 @@ static int __loc_database_enumerator_next_network_flattened(
- *network = loc_network_list_pop_first(subnets);
-
- // Push the rest onto the stack
-- loc_network_list_reverse(subnets);
-- loc_network_list_merge(enumerator->stack, subnets);
-+ loc_network_list_merge_reverse(enumerator->stack, subnets);
-
- loc_network_list_unref(subnets);
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index 406dd15..56cada8 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -140,6 +140,7 @@ global:
- loc_network_list_empty;
- loc_network_list_get;
- loc_network_list_merge;
-+ loc_network_list_merge_reverse;
- loc_network_list_new;
- loc_network_list_pop;
- loc_network_list_pop_first;
-diff --git a/src/loc/network-list.h b/src/loc/network-list.h
-index af3b28d..89776a6 100644
---- a/src/loc/network-list.h
-+++ b/src/loc/network-list.h
-@@ -33,5 +33,6 @@ int loc_network_list_contains(struct loc_network_list* list, struct loc_network*
- void loc_network_list_sort(struct loc_network_list* list);
- void loc_network_list_reverse(struct loc_network_list* list);
- int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
-+int loc_network_list_merge_reverse(struct loc_network_list* self, struct loc_network_list* other);
+- DELETE FROM networks WHERE EXISTS (
+- SELECT FROM _garbage WHERE networks.network = _garbage.network
+- );
++ ON CONFLICT (number) DO UPDATE SET name = excluded.name;
+ """)
- #endif
-diff --git a/src/network-list.c b/src/network-list.c
-index 9cb4547..b455caf 100644
---- a/src/network-list.c
-+++ b/src/network-list.c
-@@ -242,3 +242,16 @@ LOC_EXPORT int loc_network_list_merge(
+ # Download all extended sources
+@@ -405,6 +469,69 @@ class CLI(object):
+ for line in f:
+ self._parse_line(line)
- return 0;
- }
++ def _check_parsed_network(self, network):
++ """
++ Assistive function to detect and subsequently sort out parsed
++ networks from RIR data (both Whois and so-called "extended sources"),
++ which are or have...
+
-+LOC_EXPORT int loc_network_list_merge_reverse(
-+ struct loc_network_list* self, struct loc_network_list* other) {
-+ int r;
++ (a) not globally routable (RFC 1918 space, et al.)
++ (b) covering a too large chunk of the IP address space (prefix length
++ is < 7 for IPv4 networks, and < 10 for IPv6)
++ (c) "0.0.0.0" or "::" as a network address
++ (d) are too small for being publicly announced (we have decided not to
++ process them at the moment, as they significantly enlarge our
++ database without providing very helpful additional information)
+
-+ for (int i = other->size - 1; i >= 0; i--) {
-+ r = loc_network_list_push(self, other->elements[i]);
-+ if (r)
-+ return r;
-+ }
++ This unfortunately is necessary due to brain-dead clutter across
++ various RIR databases, causing mismatches and eventually disruptions.
+
-+ return 0;
-+}
---
-2.20.1
-
-From a5967330d530504db401540d4bcd5474fe00e421 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 20 Nov 2020 18:36:48 +0000
-Subject: [PATCH 083/111] network: Optimise _subnet function
-
-This function used to create a network list which always
-had exactly two elements. Since splitting a network in half
-always returns two parts, we can simply return them as a
-pointer.
-
-This improves returning the network tree by about 17%.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/loc/network.h | 2 +-
- src/network.c | 158 ++++++++++++++++++---------------------------
- src/test-network.c | 51 ++++++++-------
- 3 files changed, 91 insertions(+), 120 deletions(-)
-
-diff --git a/src/loc/network.h b/src/loc/network.h
-index 4b7410c..d67ec54 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -62,7 +62,7 @@ int loc_network_gt(struct loc_network* self, struct loc_network* other);
- int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
- int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
- int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
--struct loc_network_list* loc_network_subnets(struct loc_network* network);
-+int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
- struct loc_network_list* loc_network_exclude(
- struct loc_network* self, struct loc_network* other);
- struct loc_network_list* loc_network_exclude_list(
-diff --git a/src/network.c b/src/network.c
-index 3271272..ff9c1eb 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -507,66 +507,91 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net
- return loc_network_is_subnet(other, self);
- }
-
--LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
-- struct loc_network_list* list;
-+LOC_EXPORT int loc_network_subnets(struct loc_network* network,
-+ struct loc_network** subnet1, struct loc_network** subnet2) {
-+ int r;
-+ *subnet1 = NULL;
-+ *subnet2 = NULL;
-
- // New prefix length
- unsigned int prefix = network->prefix + 1;
-
- // Check if the new prefix is valid
- if (valid_prefix(&network->first_address, prefix))
-- return NULL;
--
-- // Create a new list with the result
-- int r = loc_network_list_new(network->ctx, &list);
-- if (r) {
-- ERROR(network->ctx, "Could not create network list: %d\n", r);
-- return NULL;
-- }
--
-- struct loc_network* subnet1 = NULL;
-- struct loc_network* subnet2 = NULL;
-+ return -1;
-
- // Create the first half of the network
-- r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix);
-+ r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix);
- if (r)
-- goto ERROR;
-+ return r;
-
- // The next subnet starts after the first one
-- struct in6_addr first_address = address_increment(&subnet1->last_address);
-+ struct in6_addr first_address = address_increment(&(*subnet1)->last_address);
-
- // Create the second half of the network
-- r = loc_network_new(network->ctx, &subnet2, &first_address, prefix);
-- if (r)
-- goto ERROR;
--
-- // Push the both onto the stack (in reverse order)
-- r = loc_network_list_push(list, subnet2);
-+ r = loc_network_new(network->ctx, subnet2, &first_address, prefix);
- if (r)
-- goto ERROR;
--
-- r = loc_network_list_push(list, subnet1);
-- if (r)
-- goto ERROR;
-+ return r;
-
- // Copy country code
- const char* country_code = loc_network_get_country_code(network);
- if (country_code) {
-- loc_network_set_country_code(subnet1, country_code);
-- loc_network_set_country_code(subnet2, country_code);
-+ loc_network_set_country_code(*subnet1, country_code);
-+ loc_network_set_country_code(*subnet2, country_code);
- }
-
- // Copy ASN
- uint32_t asn = loc_network_get_asn(network);
- if (asn) {
-- loc_network_set_asn(subnet1, asn);
-- loc_network_set_asn(subnet2, asn);
-+ loc_network_set_asn(*subnet1, asn);
-+ loc_network_set_asn(*subnet2, asn);
- }
-
-- loc_network_unref(subnet1);
-- loc_network_unref(subnet2);
-+ return 0;
-+}
-
-- return list;
-+static int __loc_network_exclude(struct loc_network* network,
-+ struct loc_network* other, struct loc_network_list* list) {
-+ struct loc_network* subnet1 = NULL;
-+ struct loc_network* subnet2 = NULL;
++ We will return False in case a network is not suitable for adding
++ it to our database, and True otherwise.
++ """
+
-+ int r = loc_network_subnets(network, &subnet1, &subnet2);
-+ if (r)
-+ goto ERROR;
++ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
++ return False
+
-+ if (loc_network_eq(other, subnet1)) {
-+ r = loc_network_list_push(list, subnet2);
-+ if (r)
-+ goto ERROR;
++ if not network.is_global:
++ log.warning("Skipping non-globally routable network: %s" % network)
++ return False
+
-+ } else if (loc_network_eq(other, subnet2)) {
-+ r = loc_network_list_push(list, subnet1);
-+ if (r)
-+ goto ERROR;
++ if network.version == 4:
++ if network.prefixlen < 7:
++ log.warning("Skipping too big IP chunk: %s" % network)
++ return False
+
-+ } else if (loc_network_is_subnet_of(other, subnet1)) {
-+ r = loc_network_list_push(list, subnet2);
-+ if (r)
-+ goto ERROR;
++ if network.prefixlen > 24:
++ log.debug("Skipping network too small to be publicly announced: %s" % network)
++ return False
+
-+ r = __loc_network_exclude(subnet1, other, list);
-+ if (r)
-+ goto ERROR;
++ if str(network.network_address) == "0.0.0.0":
++ log.warning("Skipping network based on 0.0.0.0: %s" % network)
++ return False
+
-+ } else if (loc_network_is_subnet_of(other, subnet2)) {
-+ r = loc_network_list_push(list, subnet1);
-+ if (r)
-+ goto ERROR;
++ elif network.version == 6:
++ if network.prefixlen < 10:
++ log.warning("Skipping too big IP chunk: %s" % network)
++ return False
+
-+ r = __loc_network_exclude(subnet2, other, list);
-+ if (r)
-+ goto ERROR;
++ if network.prefixlen > 48:
++ log.debug("Skipping network too small to be publicly announced: %s" % network)
++ return False
+
-+ } else {
-+ ERROR(network->ctx, "We should never get here\n");
-+ r = 1;
-+ goto ERROR;
-+ }
-
- ERROR:
- if (subnet1)
-@@ -575,10 +600,7 @@ ERROR:
- if (subnet2)
- loc_network_unref(subnet2);
-
-- if (list)
-- loc_network_list_unref(list);
--
-- return NULL;
-+ return r;
- }
-
- LOC_EXPORT struct loc_network_list* loc_network_exclude(
-@@ -623,67 +645,15 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude(
- return NULL;
- }
-
-- struct loc_network_list* subnets = loc_network_subnets(self);
--
-- struct loc_network* subnet1 = NULL;
-- struct loc_network* subnet2 = NULL;
--
-- while (subnets) {
-- // Fetch both subnets
-- subnet1 = loc_network_list_get(subnets, 0);
-- subnet2 = loc_network_list_get(subnets, 1);
--
-- // Free list
-- loc_network_list_unref(subnets);
-- subnets = NULL;
--
-- if (loc_network_eq(other, subnet1)) {
-- r = loc_network_list_push(list, subnet2);
-- if (r)
-- goto ERROR;
--
-- } else if (loc_network_eq(other, subnet2)) {
-- r = loc_network_list_push(list, subnet1);
-- if (r)
-- goto ERROR;
--
-- } else if (loc_network_is_subnet_of(other, subnet1)) {
-- r = loc_network_list_push(list, subnet2);
-- if (r)
-- goto ERROR;
--
-- subnets = loc_network_subnets(subnet1);
--
-- } else if (loc_network_is_subnet_of(other, subnet2)) {
-- r = loc_network_list_push(list, subnet1);
-- if (r)
-- goto ERROR;
--
-- subnets = loc_network_subnets(subnet2);
--
-- } else {
-- ERROR(self->ctx, "We should never get here\n");
-- goto ERROR;
-- }
-+ r = __loc_network_exclude(self, other, list);
-+ if (r) {
-+ loc_network_list_unref(list);
++ if str(network.network_address) == "::":
++ log.warning("Skipping network based on '::': %s" % network)
++ return False
++
++ else:
++ # This should not happen...
++ log.warning("Skipping network of unknown family, this should not happen: %s" % network)
++ return False
++
++ # In case we have made it here, the network is considered to
++ # be suitable for libloc consumption...
++ return True
++
+ def _parse_block(self, block):
+ # Get first line to find out what type of block this is
+ line = block[0]
+@@ -433,7 +560,7 @@ class CLI(object):
+ autnum["asn"] = m.group(2)
-- loc_network_unref(subnet1);
-- loc_network_unref(subnet2);
-+ return NULL;
- }
+ elif key == "org":
+- autnum[key] = val
++ autnum[key] = val.upper()
- // Return the result
- return list;
--
--ERROR:
-- if (subnet1)
-- loc_network_unref(subnet1);
--
-- if (subnet2)
-- loc_network_unref(subnet2);
--
-- if (list)
-- loc_network_list_unref(list);
--
-- return NULL;
- }
+ # Skip empty objects
+ if not autnum:
+@@ -447,15 +574,22 @@ class CLI(object):
+ )
- LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
-diff --git a/src/test-network.c b/src/test-network.c
-index 7c90224..79c2967 100644
---- a/src/test-network.c
-+++ b/src/test-network.c
-@@ -138,47 +138,48 @@ int main(int argc, char** argv) {
- }
+ def _parse_inetnum_block(self, block):
+- logging.debug("Parsing inetnum block:")
++ log.debug("Parsing inetnum block:")
- // Check subnet function
-- err = loc_network_is_subnet_of(network1, network2);
-- if (err != 0) {
-+ err = loc_network_is_subnet(network1, network2);
-+ if (!err) {
- fprintf(stderr, "Subnet check 1 failed: %d\n", err);
- exit(EXIT_FAILURE);
- }
+ inetnum = {}
+ for line in block:
+- logging.debug(line)
++ log.debug(line)
-- err = loc_network_is_subnet_of(network2, network1);
-- if (err != 1) {
-+ err = loc_network_is_subnet(network2, network1);
-+ if (err) {
- fprintf(stderr, "Subnet check 2 failed: %d\n", err);
- exit(EXIT_FAILURE);
- }
+ # Split line
+ key, val = split_line(line)
-- // Make list of subnets
-- struct loc_network_list* subnets = loc_network_subnets(network1);
-- if (!subnets) {
-- fprintf(stderr, "Could not find subnets of network\n");
-+ // Make subnets
-+ struct loc_network* subnet1 = NULL;
-+ struct loc_network* subnet2 = NULL;
++ # Filter any inetnum records which are only referring to IP space
++ # not managed by that specific RIR...
++ if key == "netname":
++ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
++ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
++ return
+
-+ err = loc_network_subnets(network1, &subnet1, &subnet2);
-+ if (err || !subnet1 || !subnet2) {
-+ fprintf(stderr, "Could not find subnets of network: %d\n", err);
- exit(EXIT_FAILURE);
- }
+ if key == "inetnum":
+ start_address, delim, end_address = val.partition("-")
+
+@@ -467,7 +601,7 @@ class CLI(object):
+ start_address = ipaddress.ip_address(start_address)
+ end_address = ipaddress.ip_address(end_address)
+ except ValueError:
+- logging.warning("Could not parse line: %s" % line)
++ log.warning("Could not parse line: %s" % line)
+ return
+
+ # Set prefix to default
+@@ -484,23 +618,24 @@ class CLI(object):
+ inetnum[key] = val
-- loc_network_list_dump(subnets);
+ elif key == "country":
+- if val == "UNITED STATES":
+- val = "US"
-
-- while (!loc_network_list_empty(subnets)) {
-- struct loc_network* subnet = loc_network_list_pop(subnets);
-- if (!subnet) {
-- fprintf(stderr, "Received an empty subnet\n");
-- exit(EXIT_FAILURE);
-- }
-+ char* s = loc_network_str(subnet1);
-+ printf("Received subnet1 = %s\n", s);
-+ free(s);
+ inetnum[key] = val.upper()
-- char* s = loc_network_str(subnet);
-- printf("Received subnet %s\n", s);
-- free(s);
-+ s = loc_network_str(subnet2);
-+ printf("Received subnet2 = %s\n", s);
-+ free(s);
+ # Skip empty objects
+- if not inetnum:
++ if not inetnum or not "country" in inetnum:
++ return
++
++ # Skip objects with bogus country code 'ZZ'
++ if inetnum.get("country") == "ZZ":
++ log.warning("Skipping network with bogus country 'ZZ': %s" % \
++ (inetnum.get("inet6num") or inetnum.get("inetnum")))
+ return
-- if (!loc_network_is_subnet_of(subnet, network1)) {
-- fprintf(stderr, "Not a subnet\n");
-- exit(EXIT_FAILURE);
-- }
-+ if (!loc_network_is_subnet(network1, subnet1)) {
-+ fprintf(stderr, "Subnet1 is not a subnet\n");
-+ exit(EXIT_FAILURE);
-+ }
+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-- loc_network_unref(subnet);
-+ if (!loc_network_is_subnet(network1, subnet2)) {
-+ fprintf(stderr, "Subnet2 is not a subnet\n");
-+ exit(EXIT_FAILURE);
- }
+- # Bail out in case we have processed a non-public IP network
+- if network.is_private:
+- logging.warning("Skipping non-globally routable network: %s" % network)
++ if not self._check_parsed_network(network):
+ return
-- loc_network_list_unref(subnets);
-+ loc_network_unref(subnet1);
-+ loc_network_unref(subnet2);
+- self.db.execute("INSERT INTO networks(network, country) \
++ self.db.execute("INSERT INTO _rirdata(network, country) \
+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+ "%s" % network, inetnum.get("country"),
+ )
+@@ -511,7 +646,9 @@ class CLI(object):
+ # Split line
+ key, val = split_line(line)
- struct loc_network_list* excluded = loc_network_exclude(network1, network2);
- if (!excluded) {
---
-2.20.1
-
-From 7fe6a21845edf6692d239f228117bc95620d0419 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 20 Nov 2020 18:38:47 +0000
-Subject: [PATCH 084/111] network: Add function to return the prefix
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 1 +
- src/loc/network.h | 1 +
- src/network.c | 12 ++++++++++++
- 3 files changed, 14 insertions(+)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index 56cada8..9f41c89 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -125,6 +125,7 @@ global:
- loc_network_new;
- loc_network_new_from_string;
- loc_network_overlaps;
-+ loc_network_prefix;
- loc_network_ref;
- loc_network_set_asn;
- loc_network_set_country_code;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index d67ec54..7b2ae4c 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -38,6 +38,7 @@ struct loc_network* loc_network_ref(struct loc_network* network);
- struct loc_network* loc_network_unref(struct loc_network* network);
- char* loc_network_str(struct loc_network* network);
- int loc_network_address_family(struct loc_network* network);
-+unsigned int loc_network_prefix(struct loc_network* network);
+- if key in ("organisation", "org-name"):
++ if key == "organisation":
++ org[key] = val.upper()
++ elif key == "org-name":
+ org[key] = val
- const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
- char* loc_network_format_first_address(struct loc_network* network);
-diff --git a/src/network.c b/src/network.c
-index ff9c1eb..4720503 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -310,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) {
- return network->family;
- }
+ # Skip empty objects
+@@ -581,6 +718,9 @@ class CLI(object):
+ log.warning("Invalid IP address: %s" % address)
+ return
-+LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) {
-+ switch (network->family) {
-+ case AF_INET6:
-+ return network->prefix;
-+
-+ case AF_INET:
-+ return network->prefix - 96;
-+ }
-+
-+ return 0;
-+}
++ if not self._check_parsed_network(network):
++ return
+
- static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) {
- const size_t length = INET6_ADDRSTRLEN;
-
---
-2.20.1
-
-From cf8d3c6454843943e3bc81eb85522779d1c11f9b Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 20 Nov 2020 18:39:13 +0000
-Subject: [PATCH 085/111] network: Speed up subnet check
-
-There is no point in checking different address families
-with each other and we do not need to compare addresses
-when the prefix of the subnet does not fit into the
-network to check.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 8 ++++++++
- 1 file changed, 8 insertions(+)
-
-diff --git a/src/network.c b/src/network.c
-index 4720503..e52b58c 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -501,6 +501,14 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
- }
+ self.db.execute("INSERT INTO networks(network, country) \
+ VALUES(%s, %s) ON CONFLICT (network) DO \
+ UPDATE SET country = excluded.country",
+diff --git a/src/python/location.in b/src/python/location.in
+index 44ad726..b30beae 100644
+--- a/src/python/location.in
++++ b/src/python/location.in
+@@ -253,6 +253,7 @@ class CLI(object):
+ network = db.lookup(address)
+ except ValueError:
+ print(_("Invalid IP address: %s") % address, file=sys.stderr)
++ return 2
- LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
-+ // Check family
-+ if (self->family != other->family)
-+ return 0;
-+
-+ // The prefix must be smaller (this avoids the more complex comparisons later)
-+ if (self->prefix > other->prefix)
-+ return 0;
-+
- // If the start address of the other network is smaller than this network,
- // it cannot be a subnet.
- if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
---
-2.20.1
-
-From d6a5092f969bb3bd50d130d4ba64b4e4be2e61f6 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri, 20 Nov 2020 18:44:42 +0000
-Subject: [PATCH 086/111] network: Remove deprecated loc_network_is_subnet_of
- function
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 1 -
- src/loc/network.h | 1 -
- src/network.c | 13 ++++---------
- src/python/network.c | 2 +-
- 4 files changed, 5 insertions(+), 12 deletions(-)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index 9f41c89..b2d8a31 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -118,7 +118,6 @@ global:
- loc_network_gt;
- loc_network_has_flag;
- loc_network_is_subnet;
-- loc_network_is_subnet_of;
- loc_network_match_asn;
- loc_network_match_country_code;
- loc_network_match_flag;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index 7b2ae4c..b31c8a2 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -62,7 +62,6 @@ int loc_network_eq(struct loc_network* self, struct loc_network* other);
- int loc_network_gt(struct loc_network* self, struct loc_network* other);
- int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
- int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
--int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
- int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
- struct loc_network_list* loc_network_exclude(
- struct loc_network* self, struct loc_network* other);
-diff --git a/src/network.c b/src/network.c
-index e52b58c..72b77e6 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -522,11 +522,6 @@ LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_networ
- return 1;
- }
+ args = {
+ "address" : address,
+@@ -398,10 +399,7 @@ class CLI(object):
--// XXX DEPRECATED - I find this too difficult to use
--LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
-- return loc_network_is_subnet(other, self);
--}
+ def handle_update(self, db, ns):
+ if ns.cron and db:
+- now = datetime.datetime.utcnow()
-
- LOC_EXPORT int loc_network_subnets(struct loc_network* network,
- struct loc_network** subnet1, struct loc_network** subnet2) {
- int r;
-@@ -589,7 +584,7 @@ static int __loc_network_exclude(struct loc_network* network,
- if (r)
- goto ERROR;
-
-- } else if (loc_network_is_subnet_of(other, subnet1)) {
-+ } else if (loc_network_is_subnet(subnet1, other)) {
- r = loc_network_list_push(list, subnet2);
- if (r)
- goto ERROR;
-@@ -598,7 +593,7 @@ static int __loc_network_exclude(struct loc_network* network,
- if (r)
- goto ERROR;
-
-- } else if (loc_network_is_subnet_of(other, subnet2)) {
-+ } else if (loc_network_is_subnet(subnet2, other)) {
- r = loc_network_list_push(list, subnet1);
- if (r)
- goto ERROR;
-@@ -645,7 +640,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude(
- }
-
- // Other must be a subnet of self
-- if (!loc_network_is_subnet_of(other, self)) {
-+ if (!loc_network_is_subnet(self, other)) {
- DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
+- # Parse the database timestamp
+- t = datetime.datetime.utcfromtimestamp(db.created_at)
++ now = time.time()
- return NULL;
-@@ -726,7 +721,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
- }
+ if ns.cron == "daily":
+ delta = datetime.timedelta(days=1)
+@@ -410,22 +408,20 @@ class CLI(object):
+ elif ns.cron == "monthly":
+ delta = datetime.timedelta(days=30)
- // Drop this subnet if is a subnet of another subnet
-- if (loc_network_is_subnet_of(subnet, subnet_to_check)) {
-+ if (loc_network_is_subnet(subnet_to_check, subnet)) {
- passed = 0;
- loc_network_unref(subnet);
- break;
-diff --git a/src/python/network.c b/src/python/network.c
-index 742b472..b6e92fb 100644
---- a/src/python/network.c
-+++ b/src/python/network.c
-@@ -194,7 +194,7 @@ static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
- if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
- return NULL;
++ delta = delta.total_seconds()
++
+ # Check if the database has recently been updated
+- if t >= (now - delta):
++ if db.created_at >= (now - delta):
+ log.info(
+- _("The database has been updated recently (%s)") % \
+- format_timedelta(now - t),
++ _("The database has been updated recently"),
+ )
+ return 3
-- if (loc_network_is_subnet_of(self->network, other->network))
-+ if (loc_network_is_subnet(other->network, self->network))
- Py_RETURN_TRUE;
+ # Fetch the timestamp we need from DNS
+ t = location.discover_latest_version()
- Py_RETURN_FALSE;
---
-2.20.1
-
-From af4689bf5c56ad79e9e90396b41be460e49ef384 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 24 Nov 2020 14:55:08 +0000
-Subject: [PATCH 087/111] network-list: Make this a sorted list
-
-The list is now organised so that it is always ordered.
-
-This allows us to find if a network is on the list a lot
-quicker using binary search as well as inserting a new
-network at the right place.
-
-This will benefit loc_network_exclude with very large networks.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 1 +
- src/loc/network.h | 1 +
- src/network-list.c | 84 +++++++++++++++++++++++++++++++++++++++++-----
- src/network.c | 22 ++++++++++++
- 4 files changed, 99 insertions(+), 9 deletions(-)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index b2d8a31..28cc8e8 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -106,6 +106,7 @@ global:
+- # Parse timestamp into datetime format
+- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
+-
+ # Check the version of the local database
+- if db and timestamp and db.created_at >= timestamp.timestamp():
++ if db and t and db.created_at >= t:
+ log.info("Already on the latest version")
+ return
- # Network
- loc_network_address_family;
-+ loc_network_cmp;
- loc_network_eq;
- loc_network_exclude;
- loc_network_exclude_list;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index b31c8a2..8ab1562 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -58,6 +58,7 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
- int loc_network_set_flag(struct loc_network* network, uint32_t flag);
- int loc_network_match_flag(struct loc_network* network, uint32_t flag);
+@@ -437,7 +433,7 @@ class CLI(object):
-+int loc_network_cmp(struct loc_network* self, struct loc_network* other);
- int loc_network_eq(struct loc_network* self, struct loc_network* other);
- int loc_network_gt(struct loc_network* self, struct loc_network* other);
- int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
-diff --git a/src/network-list.c b/src/network-list.c
-index b455caf..6e9cd37 100644
---- a/src/network-list.c
-+++ b/src/network-list.c
-@@ -16,6 +16,7 @@
+ # Try downloading a new database
+ try:
+- t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
++ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
- #include <errno.h>
- #include <stdlib.h>
-+#include <time.h>
+ # If no file could be downloaded, log a message
+ except FileNotFoundError as e:
+@@ -453,13 +449,7 @@ class CLI(object):
- #include <loc/libloc.h>
- #include <loc/network.h>
-@@ -130,11 +131,71 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
- return loc_network_ref(list->elements[index]);
- }
+ return 0
-+//MOVE FUNCTION GOES HERE
-+
-+static off_t loc_network_list_find(struct loc_network_list* list,
-+ struct loc_network* network, int* found) {
-+ off_t lo = 0;
-+ off_t hi = list->size - 1;
-+
-+ *found = 0;
-+
-+#ifdef ENABLE_DEBUG
-+ // Save start time
-+ clock_t start = clock();
-+#endif
-+
-+ off_t i = 0;
-+
-+ while (lo <= hi) {
-+ i = (lo + hi) / 2;
-+
-+ // Check if this is a match
-+ int result = loc_network_cmp(network, list->elements[i]);
-+
-+ if (result == 0) {
-+ *found = 1;
-+
-+#ifdef ENABLE_DEBUG
-+ clock_t end = clock();
+- def handle_verify(self, ns):
+- try:
+- db = location.Database(ns.database)
+- except FileNotFoundError as e:
+- log.error("%s: %s" % (ns.database, e))
+- return 127
+-
++ def handle_verify(self, db, ns):
+ # Verify the database
+ with open(ns.public_key, "r") as f:
+ if not db.verify(f):
+diff --git a/src/python/network.c b/src/python/network.c
+index 5496d1e..b6e92fb 100644
+--- a/src/python/network.c
++++ b/src/python/network.c
+@@ -20,10 +20,29 @@
+
+ #include <loc/libloc.h>
+ #include <loc/network.h>
++#include <loc/network-list.h>
+
+ #include "locationmodule.h"
+ #include "network.h"
+
++static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
++ PyObject* list = PyList_New(0);
++ if (!networks)
++ return list;
+
-+ // Log how fast this has been
-+ DEBUG(list->ctx, "Found network in %.4fms at %jd\n",
-+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
-+#endif
++ while (!loc_network_list_empty(networks)) {
++ struct loc_network* network = loc_network_list_pop(networks);
+
-+ return i;
-+ }
++ PyObject* n = new_network(&NetworkType, network);
++ PyList_Append(list, n);
+
-+ if (result > 0)
-+ lo = i + 1;
-+ else
-+ hi = i - 1;
++ loc_network_unref(network);
++ Py_DECREF(n);
+ }
+
-+#ifdef ENABLE_DEBUG
-+ clock_t end = clock();
++ return list;
++}
+
-+ // Log how fast this has been
-+ DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n",
-+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
-+#endif
+ PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
+ NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
+ if (self) {
+@@ -154,13 +173,28 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
+ Py_RETURN_NONE;
+ }
+
++static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
++ NetworkObject* other = NULL;
+
-+ return i;
-+}
++ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
++ return NULL;
+
- LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
-- // Do not add networks that are already on the list
-- if (loc_network_list_contains(list, network))
-+ int found = 0;
++ struct loc_network_list* list = loc_network_exclude(self->network, other->network);
+
-+ off_t index = loc_network_list_find(list, network, &found);
++ // Convert to Python objects
++ PyObject* obj = PyList_FromNetworkList(list);
++ loc_network_list_unref(list);
+
-+ // The network has been found on the list. Nothing to do.
-+ if (found)
- return 0;
-
-+ DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n",
-+ list, network, (intmax_t)index);
++ return obj;
++}
+
- // Check if we have space left
- if (list->size >= list->elements_size) {
- int r = loc_network_list_grow(list, 64);
-@@ -142,9 +203,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
- return r;
- }
+ static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
+ NetworkObject* other = NULL;
-- DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
-+ // The list is now larger
-+ list->size++;
-+
-+ // Move all elements out of the way
-+ for (unsigned int i = list->size - 1; i > index; i--)
-+ list->elements[i] = list->elements[i - 1];
+ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
+ return NULL;
-- list->elements[list->size++] = loc_network_ref(network);
-+ // Add the new element at the right place
-+ list->elements[index] = loc_network_ref(network);
+- if (loc_network_is_subnet_of(self->network, other->network))
++ if (loc_network_is_subnet(other->network, self->network))
+ Py_RETURN_TRUE;
- return 0;
- }
-@@ -183,12 +250,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
+ Py_RETURN_FALSE;
+@@ -181,6 +215,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) {
+ return obj;
}
- LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
-- for (unsigned int i = 0; i < list->size; i++) {
-- if (loc_network_eq(list->elements[i], network))
-- return 1;
-- }
-+ int found = 0;
-
-- return 0;
-+ loc_network_list_find(list, network, &found);
++static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) {
++ struct in_addr address4;
+
-+ return found;
- }
-
- static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
-diff --git a/src/network.c b/src/network.c
-index 72b77e6..38d557a 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -441,6 +441,28 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
- return loc_network_has_flag(network, flag);
- }
-
-+LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
-+ // Compare family
-+ if (self->family > other->family)
-+ return 1;
-+ else if (self->family < other->family)
-+ return -1;
++ // Convert IPv4 addresses to struct in_addr
++ if (IN6_IS_ADDR_V4MAPPED(address6)) {
++ address4.s_addr = address6->s6_addr32[3];
+
-+ // Compare address
-+ int r = in6_addr_cmp(&self->first_address, &other->first_address);
-+ if (r)
-+ return r;
++ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4));
++ }
+
-+ // Compare prefix
-+ if (self->prefix > other->prefix)
-+ return 1;
-+ else if (self->prefix < other->prefix)
-+ return -1;
++ // Return IPv6 addresses as they are
++ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6));
++}
+
-+ // Both networks are equal
-+ return 0;
++static PyObject* Network_get__first_address(NetworkObject* self) {
++ const struct in6_addr* address = loc_network_get_first_address(self->network);
++
++ return PyBytes_FromAddress(address);
+}
+
- LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
- // Family must be the same
- if (self->family != other->family)
---
-2.20.1
-
-From 39cbbc63aee362d82f69a9b4722b59153ce799a0 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 24 Nov 2020 15:04:03 +0000
-Subject: [PATCH 088/111] network-list: Drop sorting functions
-
-Since the list is always sorted, there is no point in
-sorting it again...
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 6 +----
- src/libloc.sym | 3 ---
- src/loc/network-list.h | 3 ---
- src/network-list.c | 52 ------------------------------------------
- src/test-network.c | 9 --------
- 5 files changed, 1 insertion(+), 72 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index ba4f98a..5546091 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1372,17 +1372,13 @@ static int __loc_database_enumerator_next_network_flattened(
- // We no longer need the excluded list
- loc_network_list_unref(excluded);
-
-- // Sort all subnets
-- loc_network_list_sort(subnets);
--
- // Replace network with the first one
- loc_network_unref(*network);
-
- *network = loc_network_list_pop_first(subnets);
-
- // Push the rest onto the stack
-- loc_network_list_merge_reverse(enumerator->stack, subnets);
--
-+ loc_network_list_merge(enumerator->stack, subnets);
- loc_network_list_unref(subnets);
-
- return 0;
-diff --git a/src/libloc.sym b/src/libloc.sym
-index 28cc8e8..4b0ce45 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -141,15 +141,12 @@ global:
- loc_network_list_empty;
- loc_network_list_get;
- loc_network_list_merge;
-- loc_network_list_merge_reverse;
- loc_network_list_new;
- loc_network_list_pop;
- loc_network_list_pop_first;
- loc_network_list_push;
- loc_network_list_ref;
-- loc_network_list_reverse;
- loc_network_list_size;
-- loc_network_list_sort;
- loc_network_list_unref;
-
- # Writer
-diff --git a/src/loc/network-list.h b/src/loc/network-list.h
-index 89776a6..21c7402 100644
---- a/src/loc/network-list.h
-+++ b/src/loc/network-list.h
-@@ -30,9 +30,6 @@ int loc_network_list_push(struct loc_network_list* list, struct loc_network* net
- struct loc_network* loc_network_list_pop(struct loc_network_list* list);
- struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
- int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
--void loc_network_list_sort(struct loc_network_list* list);
--void loc_network_list_reverse(struct loc_network_list* list);
- int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
--int loc_network_list_merge_reverse(struct loc_network_list* self, struct loc_network_list* other);
-
- #endif
-diff --git a/src/network-list.c b/src/network-list.c
-index 6e9cd37..7e8b5f3 100644
---- a/src/network-list.c
-+++ b/src/network-list.c
-@@ -257,45 +257,6 @@ LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct l
- return found;
- }
-
--static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
-- // Do nothing for invalid indices
-- if (i1 >= list->size || i2 >= list->size)
-- return;
--
-- struct loc_network* network1 = list->elements[i1];
-- struct loc_network* network2 = list->elements[i2];
--
-- list->elements[i1] = network2;
-- list->elements[i2] = network1;
--}
--
--LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
-- unsigned int i = 0;
-- unsigned int j = list->size - 1;
--
-- while (i < j) {
-- loc_network_list_swap(list, i++, j--);
-- }
--}
--
--LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
-- unsigned int n = list->size;
-- int swapped;
--
-- do {
-- swapped = 0;
--
-- for (unsigned int i = 1; i < n; i++) {
-- if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) {
-- loc_network_list_swap(list, i-1, i);
-- swapped = 1;
-- }
-- }
--
-- n--;
-- } while (swapped);
--}
--
- LOC_EXPORT int loc_network_list_merge(
- struct loc_network_list* self, struct loc_network_list* other) {
- int r;
-@@ -308,16 +269,3 @@ LOC_EXPORT int loc_network_list_merge(
+ static PyObject* Network_get_last_address(NetworkObject* self) {
+ char* address = loc_network_format_last_address(self->network);
- return 0;
+@@ -190,7 +244,19 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
+ return obj;
}
--
--LOC_EXPORT int loc_network_list_merge_reverse(
-- struct loc_network_list* self, struct loc_network_list* other) {
-- int r;
--
-- for (int i = other->size - 1; i >= 0; i--) {
-- r = loc_network_list_push(self, other->elements[i]);
-- if (r)
-- return r;
-- }
--
-- return 0;
--}
-diff --git a/src/test-network.c b/src/test-network.c
-index 79c2967..8a6763c 100644
---- a/src/test-network.c
-+++ b/src/test-network.c
-@@ -188,15 +188,6 @@ int main(int argc, char** argv) {
- }
-
- loc_network_list_dump(excluded);
--
-- // Reverse it
-- loc_network_list_reverse(excluded);
-- loc_network_list_dump(excluded);
--
-- // Sort them and dump them again
-- loc_network_list_sort(excluded);
-- loc_network_list_dump(excluded);
--
- loc_network_list_unref(excluded);
-
- // Create a database
---
-2.20.1
-
-From 5dacb45afceac2d05ea597755c1ca5a1b62cc0fd Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 24 Nov 2020 15:15:59 +0000
-Subject: [PATCH 089/111] database: Avoid merging the same data twice
-
-When finish splitting networks into many parts, we have
-a list of subnets with the excluded subnets and merge them
-together first and put them on the stack again.
-
-This is slower than pushing it all onto the stack first
-and then popping the first element.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 24 ++++++++++++++----------
- 1 file changed, 14 insertions(+), 10 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index 5546091..f1f6ae0 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1360,8 +1360,8 @@ static int __loc_database_enumerator_next_network_flattened(
- return -1;
- }
-
-- // Merge excluded list with subnets
-- r = loc_network_list_merge(subnets, excluded);
-+ // Merge subnets onto the stack
-+ r = loc_network_list_merge(enumerator->stack, subnets);
- if (r) {
- loc_network_list_unref(subnets);
- loc_network_list_unref(excluded);
-@@ -1369,17 +1369,21 @@ static int __loc_database_enumerator_next_network_flattened(
- return r;
- }
-
-- // We no longer need the excluded list
-- loc_network_list_unref(excluded);
--
-- // Replace network with the first one
-- loc_network_unref(*network);
-+ // Push excluded list onto the stack
-+ r = loc_network_list_merge(enumerator->stack, excluded);
-+ if (r) {
-+ loc_network_list_unref(subnets);
-+ loc_network_list_unref(excluded);
-
-- *network = loc_network_list_pop_first(subnets);
-+ return r;
-+ }
-- // Push the rest onto the stack
-- loc_network_list_merge(enumerator->stack, subnets);
- loc_network_list_unref(subnets);
-+ loc_network_list_unref(excluded);
++static PyObject* Network_get__last_address(NetworkObject* self) {
++ const struct in6_addr* address = loc_network_get_last_address(self->network);
+
-+ // Replace network with the first one from the stack
-+ loc_network_unref(*network);
-+ *network = loc_network_list_pop_first(enumerator->stack);
-
- return 0;
- }
---
-2.20.1
-
-From 04cbd2bfa892fc7374ad506ec7ba81727c5a4b96 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 24 Nov 2020 15:22:02 +0000
-Subject: [PATCH 090/111] database: Read the first element from the list
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/src/database.c b/src/database.c
-index f1f6ae0..914ed3e 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1191,7 +1191,7 @@ static int __loc_database_enumerator_next_network(
- struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
- // Return top element from the stack
- while (1) {
-- *network = loc_network_list_pop(enumerator->stack);
-+ *network = loc_network_list_pop_first(enumerator->stack);
-
- // Stack is empty
- if (!*network)
---
-2.20.1
-
-From c650008e2dd9af5fd1eadba6354aa0b615047f84 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 24 Nov 2020 15:41:53 +0000
-Subject: [PATCH 091/111] network: Add excluded networks to to_check list
-
-This is now being done immediately instead of creating a new
-list which is being merged later.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 55 ++++++++++++++++++++++++++++++---------------------
- 1 file changed, 32 insertions(+), 23 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index 38d557a..7ab22f8 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -640,49 +640,55 @@ ERROR:
- return r;
- }
-
--LOC_EXPORT struct loc_network_list* loc_network_exclude(
-- struct loc_network* self, struct loc_network* other) {
-- struct loc_network_list* list;
--
--#ifdef ENABLE_DEBUG
-- char* n1 = loc_network_str(self);
-- char* n2 = loc_network_str(other);
--
-- DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
--
-- free(n1);
-- free(n2);
--#endif
--
-+static int __loc_network_exclude_to_list(struct loc_network* self,
-+ struct loc_network* other, struct loc_network_list* list) {
- // Family must match
- if (self->family != other->family) {
- DEBUG(self->ctx, "Family mismatch\n");
-
-- return NULL;
-+ return 1;
- }
-
- // Other must be a subnet of self
- if (!loc_network_is_subnet(self, other)) {
- DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
++ return PyBytes_FromAddress(address);
++}
++
+ static struct PyMethodDef Network_methods[] = {
++ {
++ "exclude",
++ (PyCFunction)Network_exclude,
++ METH_VARARGS,
++ NULL,
++ },
+ {
+ "has_flag",
+ (PyCFunction)Network_has_flag,
+@@ -241,6 +307,13 @@ static struct PyGetSetDef Network_getsetters[] = {
+ NULL,
+ NULL,
+ },
++ {
++ "_first_address",
++ (getter)Network_get__first_address,
++ NULL,
++ NULL,
++ NULL,
++ },
+ {
+ "last_address",
+ (getter)Network_get_last_address,
+@@ -248,6 +321,13 @@ static struct PyGetSetDef Network_getsetters[] = {
+ NULL,
+ NULL,
+ },
++ {
++ "_last_address",
++ (getter)Network_get__last_address,
++ NULL,
++ NULL,
++ NULL,
++ },
+ { NULL },
+ };
-- return NULL;
-+ return 1;
- }
+diff --git a/src/test-as.c b/src/test-as.c
+index 839a04c..2d61675 100644
+--- a/src/test-as.c
++++ b/src/test-as.c
+@@ -95,7 +95,7 @@ int main(int argc, char** argv) {
+ // Enumerator
- // We cannot perform this operation if both networks equal
- if (loc_network_eq(self, other)) {
- DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
+ struct loc_database_enumerator* enumerator;
+- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
+ if (err) {
+ fprintf(stderr, "Could not create a database enumerator\n");
+ exit(EXIT_FAILURE);
+diff --git a/src/test-database.c b/src/test-database.c
+index b4a75c4..da4b11c 100644
+--- a/src/test-database.c
++++ b/src/test-database.c
+@@ -38,6 +38,14 @@ const char* DESCRIPTION =
+ "Maecenas ut venenatis nunc.";
+ const char* LICENSE = "CC";
-- return NULL;
-+ return 1;
++const char* networks[] = {
++ "2001:db8::/32",
++ "2001:db8:1000::/48",
++ "2001:db8:2000::/48",
++ "2001:db8:2020::/48",
++ NULL,
++};
++
+ static int attempt_to_open(struct loc_ctx* ctx, char* path) {
+ FILE* f = fopen(path, "r");
+ if (!f)
+@@ -139,6 +147,24 @@ int main(int argc, char** argv) {
+ exit(EXIT_FAILURE);
}
-+ return __loc_network_exclude(self, other, list);
-+}
-+
-+LOC_EXPORT struct loc_network_list* loc_network_exclude(
-+ struct loc_network* self, struct loc_network* other) {
-+ struct loc_network_list* list;
-+
-+#ifdef ENABLE_DEBUG
-+ char* n1 = loc_network_str(self);
-+ char* n2 = loc_network_str(other);
++ struct loc_network* network = NULL;
+
-+ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
++ // Add some networks
++ const char** n = networks;
++ while (*n) {
++ err = loc_writer_add_network(writer, &network, *n);
++ if (err) {
++ fprintf(stderr, "Could not add network %s\n", *n);
++ exit(EXIT_FAILURE);
++ }
+
-+ free(n1);
-+ free(n2);
-+#endif
++ // Set a country
++ loc_network_set_country_code(network, "XX");
+
- // Create a new list with the result
- int r = loc_network_list_new(self->ctx, &list);
- if (r) {
- ERROR(self->ctx, "Could not create network list: %d\n", r);
++ // Next one
++ n++;
++ }
+
- return NULL;
+ FILE* f = tmpfile();
+ if (!f) {
+ fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
+@@ -170,6 +196,33 @@ int main(int argc, char** argv) {
+ exit(EXIT_FAILURE);
}
-- r = __loc_network_exclude(self, other, list);
-+ r = __loc_network_exclude_to_list(self, other, list);
- if (r) {
- loc_network_list_unref(list);
-
-@@ -709,11 +715,14 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
- subnet = loc_network_list_get(list, i);
-
- // Find all excluded networks
-- struct loc_network_list* excluded = loc_network_exclude(network, subnet);
-- if (excluded) {
-- // Add them all to the "to check" list
-- loc_network_list_merge(to_check, excluded);
-- loc_network_list_unref(excluded);
-+ if (!loc_network_list_contains(to_check, subnet)) {
-+ r = __loc_network_exclude_to_list(network, subnet, to_check);
-+ if (r) {
-+ loc_network_list_unref(to_check);
-+ loc_network_unref(subnet);
++ // Enumerator
++ struct loc_database_enumerator* enumerator;
++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
++ if (err) {
++ fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
++ exit(EXIT_FAILURE);
++ }
+
-+ return NULL;
-+ }
- }
-
- // Cleanup
---
-2.20.1
-
-From 7c6983ad52724d395446bdbd24d36b2ce22aecfd Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 24 Nov 2020 15:42:54 +0000
-Subject: [PATCH 092/111] test: Add more networks to list to see the algorithm
- work
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/test-network-list.c | 32 +++++++++++++++++++++++++++++++-
- 1 file changed, 31 insertions(+), 1 deletion(-)
-
++ // Walk through all networks
++ while (1) {
++ err = loc_database_enumerator_next_network(enumerator, &network);
++ if (err) {
++ fprintf(stderr, "Error fetching the next network: %d\n", err);
++ exit(EXIT_FAILURE);
++ }
++
++ if (!network)
++ break;
++
++ char* s = loc_network_str(network);
++ printf("Got network: %s\n", s);
++ free(s);
++ }
++
++ // Free the enumerator
++ loc_database_enumerator_unref(enumerator);
++
+ // Close the database
+ loc_database_unref(db);
+ loc_unref(ctx);
diff --git a/src/test-network-list.c b/src/test-network-list.c
-index 3061d63..8253fc7 100644
---- a/src/test-network-list.c
+new file mode 100644
+index 0000000..6f32ff7
+--- /dev/null
+++ b/src/test-network-list.c
-@@ -58,6 +58,20 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
+@@ -0,0 +1,183 @@
++/*
++ libloc - A library to determine the location of someone on the Internet
++
++ Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
++
++ This program is free software; you can redistribute it and/or modify
++ it under the terms of the GNU General Public License as published by
++ the Free Software Foundation; either version 2 of the License, or
++ (at your option) any later version.
++
++ This program is distributed in the hope that it will be useful,
++ but WITHOUT ANY WARRANTY; without even the implied warranty of
++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
++ GNU General Public License for more details.
++*/
++
++#include <errno.h>
++#include <stdio.h>
++#include <stddef.h>
++#include <stdlib.h>
++#include <string.h>
++#include <syslog.h>
++
++#include <loc/libloc.h>
++#include <loc/network.h>
++#include <loc/network-list.h>
++
++int main(int argc, char** argv) {
++ int err;
++
++ struct loc_ctx* ctx;
++ err = loc_new(&ctx);
++ if (err < 0)
++ exit(EXIT_FAILURE);
++
++ // Enable debug logging
++ loc_set_log_priority(ctx, LOG_DEBUG);
++
++ // Create a network
++ struct loc_network* network1;
++ err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32");
++ if (err) {
++ fprintf(stderr, "Could not create the network1\n");
++ exit(EXIT_FAILURE);
++ }
++
++ struct loc_network* subnet1;
++ err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48");
++ if (err) {
++ fprintf(stderr, "Could not create the subnet1\n");
++ exit(EXIT_FAILURE);
++ }
++
++ struct loc_network* subnet2;
++ err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48");
++ if (err) {
++ fprintf(stderr, "Could not create the subnet2\n");
++ exit(EXIT_FAILURE);
++ }
++
+ struct loc_network* subnet3;
+ err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48");
+ if (err) {
+ exit(EXIT_FAILURE);
+ }
+
- // Make a list with both subnets
- struct loc_network_list* subnets;
- err = loc_network_list_new(ctx, &subnets);
-@@ -89,8 +103,24 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
-+ // Add the fourth one next
-+ err = loc_network_list_push(subnets, subnet4);
-+ if (err) {
-+ fprintf(stderr, "Could not add subnet4 to subnets list\n");
++ struct loc_network* subnet5;
++ err = loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48");
++ if (err) {
++ fprintf(stderr, "Could not create the subnet5\n");
++ exit(EXIT_FAILURE);
++ }
++
++ struct loc_network* subnet6;
++ err = loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48");
++ if (err) {
++ fprintf(stderr, "Could not create the subnet6\n");
++ exit(EXIT_FAILURE);
++ }
++
++ // Make a list with both subnets
++ struct loc_network_list* subnets;
++ err = loc_network_list_new(ctx, &subnets);
++ if (err) {
++ fprintf(stderr, "Could not create subnets list\n");
++ exit(EXIT_FAILURE);
++ }
++
++ size_t size = loc_network_list_size(subnets);
++ if (size > 0) {
++ fprintf(stderr, "The list is not empty: %zu\n", size);
++ exit(EXIT_FAILURE);
++ }
++
++ err = loc_network_list_push(subnets, subnet1);
++ if (err) {
++ fprintf(stderr, "Could not add subnet1 to subnets list\n");
++ exit(EXIT_FAILURE);
++ }
++
++ if (loc_network_list_empty(subnets)) {
++ fprintf(stderr, "The subnets list reports that it is empty\n");
++ exit(EXIT_FAILURE);
++ }
++
++ err = loc_network_list_push(subnets, subnet2);
++ if (err) {
++ fprintf(stderr, "Could not add subnet2 to subnets list\n");
++ exit(EXIT_FAILURE);
++ }
++
++ // Add the fourth one next
++ err = loc_network_list_push(subnets, subnet4);
++ if (err) {
++ fprintf(stderr, "Could not add subnet4 to subnets list\n");
++ exit(EXIT_FAILURE);
++ }
++
++ // Add the third one
++ err = loc_network_list_push(subnets, subnet3);
++ if (err) {
++ fprintf(stderr, "Could not add subnet3 to subnets list\n");
++ exit(EXIT_FAILURE);
++ }
++
++ // Add more subnets
++ err = loc_network_list_push(subnets, subnet5);
++ if (err) {
++ fprintf(stderr, "Could not add subnet5 to subnets list\n");
++ exit(EXIT_FAILURE);
++ }
++
++ err = loc_network_list_push(subnets, subnet6);
++ if (err) {
++ fprintf(stderr, "Could not add subnet6 to subnets list\n");
++ exit(EXIT_FAILURE);
++ }
++
++ loc_network_list_dump(subnets);
++
++ size = loc_network_list_size(subnets);
++ if (size != 6) {
++ fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
++ exit(EXIT_FAILURE);
++ }
++
++ // Exclude subnet1 from network1
++ struct loc_network_list* excluded = loc_network_exclude(network1, subnet1);
++ if (!excluded) {
++ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n");
+ exit(EXIT_FAILURE);
+ }
+
-+ // Add the third one
-+ err = loc_network_list_push(subnets, subnet3);
-+ if (err) {
-+ fprintf(stderr, "Could not add subnet3 to subnets list\n");
++ loc_network_list_dump(excluded);
++
++ // Exclude all subnets from network1
++ excluded = loc_network_exclude_list(network1, subnets);
++ if (!excluded) {
++ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n");
+ exit(EXIT_FAILURE);
+ }
+
-+ loc_network_list_dump(subnets);
++ loc_network_list_dump(excluded);
+
- size = loc_network_list_size(subnets);
-- if (size != 2) {
-+ if (size != 4) {
- fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
- exit(EXIT_FAILURE);
- }
---
-2.20.1
-
-From da101d55e66ebaeeed8b0b16829a2022a1af0678 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 24 Nov 2020 15:48:55 +0000
-Subject: [PATCH 093/111] Drop loc_network_eq in favour of loc_network_cmp
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 1 -
- src/loc/network.h | 1 -
- src/network.c | 24 ++++--------------------
- src/test-network.c | 8 ++++----
- 4 files changed, 8 insertions(+), 26 deletions(-)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index 4b0ce45..d8e8f14 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -107,7 +107,6 @@ global:
- # Network
- loc_network_address_family;
- loc_network_cmp;
-- loc_network_eq;
- loc_network_exclude;
- loc_network_exclude_list;
- loc_network_format_first_address;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index 8ab1562..d5d0ccd 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -59,7 +59,6 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
- int loc_network_match_flag(struct loc_network* network, uint32_t flag);
-
- int loc_network_cmp(struct loc_network* self, struct loc_network* other);
--int loc_network_eq(struct loc_network* self, struct loc_network* other);
- int loc_network_gt(struct loc_network* self, struct loc_network* other);
- int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
- int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
-diff --git a/src/network.c b/src/network.c
-index 7ab22f8..503bf3d 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -463,22 +463,6 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth
- return 0;
- }
-
--LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
-- // Family must be the same
-- if (self->family != other->family)
-- return 0;
--
-- // The start address must be the same
-- if (in6_addr_cmp(&self->first_address, &other->first_address) != 0)
-- return 0;
--
-- // The prefix length must be the same
-- if (self->prefix != other->prefix)
-- return 0;
--
-- return 1;
--}
--
- LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
- // Families must match
- if (self->family != other->family)
-@@ -596,12 +580,12 @@ static int __loc_network_exclude(struct loc_network* network,
- if (r)
- goto ERROR;
-
-- if (loc_network_eq(other, subnet1)) {
-+ if (loc_network_cmp(other, subnet1) == 0) {
- r = loc_network_list_push(list, subnet2);
- if (r)
- goto ERROR;
-
-- } else if (loc_network_eq(other, subnet2)) {
-+ } else if (loc_network_cmp(other, subnet2) == 0) {
- r = loc_network_list_push(list, subnet1);
- if (r)
- goto ERROR;
-@@ -657,7 +641,7 @@ static int __loc_network_exclude_to_list(struct loc_network* self,
- }
-
- // We cannot perform this operation if both networks equal
-- if (loc_network_eq(self, other)) {
-+ if (loc_network_cmp(self, other) == 0) {
- DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
-
- return 1;
-@@ -745,7 +729,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
- subnet = loc_network_list_get(list, i);
-
- // Drop this subnet if is is already in list
-- if (loc_network_eq(subnet_to_check, subnet)) {
-+ if (loc_network_cmp(subnet_to_check, subnet) == 0) {
- passed = 0;
- loc_network_unref(subnet);
- break;
-diff --git a/src/test-network.c b/src/test-network.c
-index 8a6763c..f4cf97b 100644
---- a/src/test-network.c
-+++ b/src/test-network.c
-@@ -125,14 +125,14 @@ int main(int argc, char** argv) {
- #endif
-
- // Check equals function
-- err = loc_network_eq(network1, network1);
-- if (!err) {
-+ err = loc_network_cmp(network1, network1);
-+ if (err) {
- fprintf(stderr, "Network is not equal with itself\n");
- exit(EXIT_FAILURE);
- }
-
-- err = loc_network_eq(network1, network2);
-- if (err) {
-+ err = loc_network_cmp(network1, network2);
-+ if (!err) {
- fprintf(stderr, "Networks equal unexpectedly\n");
- exit(EXIT_FAILURE);
- }
---
-2.20.1
-
-From 61a6f6e4bf4493236d796e94f3d721bcf5ba68aa Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 24 Nov 2020 15:50:39 +0000
-Subject: [PATCH 094/111] Drop loc_network_gt which is now unused
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/libloc.sym | 1 -
- src/loc/network.h | 1 -
- src/network.c | 27 ---------------------------
- 3 files changed, 29 deletions(-)
-
-diff --git a/src/libloc.sym b/src/libloc.sym
-index d8e8f14..cb5e8ef 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -115,7 +115,6 @@ global:
- loc_network_get_country_code;
- loc_network_get_first_address;
- loc_network_get_last_address;
-- loc_network_gt;
- loc_network_has_flag;
- loc_network_is_subnet;
- loc_network_match_asn;
-diff --git a/src/loc/network.h b/src/loc/network.h
-index d5d0ccd..af3dafd 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -59,7 +59,6 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
- int loc_network_match_flag(struct loc_network* network, uint32_t flag);
-
- int loc_network_cmp(struct loc_network* self, struct loc_network* other);
--int loc_network_gt(struct loc_network* self, struct loc_network* other);
- int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
- int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
- int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
-diff --git a/src/network.c b/src/network.c
-index 503bf3d..ac478d5 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -463,33 +463,6 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth
- return 0;
- }
-
--LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
-- // Families must match
-- if (self->family != other->family)
-- return -1;
--
-- int r = in6_addr_cmp(&self->first_address, &other->first_address);
--
-- switch (r) {
-- // Smaller
-- case -1:
-- return 0;
--
-- // Larger
-- case 1:
-- return 1;
--
-- default:
-- break;
-- }
--
-- if (self->prefix > other->prefix)
-- return 1;
--
-- // Dunno
-- return 0;
--}
--
- LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
- if (loc_network_match_address(self, &other->first_address) == 0)
- return 1;
---
-2.20.1
-
-From fc692a58d9f4ca958f88cfa202250c572a0af6ea Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 24 Nov 2020 16:50:17 +0000
-Subject: [PATCH 095/111] network: Adjust return codes of
- loc_network_match_address and add test
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/database.c | 3 +--
- src/libloc.sym | 1 +
- src/network.c | 16 +++++++++-------
- src/test-network.c | 14 ++++++++++++++
- 4 files changed, 25 insertions(+), 9 deletions(-)
-
-diff --git a/src/database.c b/src/database.c
-index 914ed3e..1871b74 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -776,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru
- }
-
- // Check if the given IP address is inside the network
-- r = loc_network_match_address(*network, address);
-- if (r) {
-+ if (!loc_network_match_address(*network, address)) {
- DEBUG(db->ctx, "Searched address is not part of the network\n");
-
- loc_network_unref(*network);
-diff --git a/src/libloc.sym b/src/libloc.sym
-index cb5e8ef..ee333f1 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -117,6 +117,7 @@ global:
- loc_network_get_last_address;
- loc_network_has_flag;
- loc_network_is_subnet;
-+ loc_network_match_address;
- loc_network_match_asn;
- loc_network_match_country_code;
- loc_network_match_flag;
-diff --git a/src/network.c b/src/network.c
-index ac478d5..febab95 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -374,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
- LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) {
- // Address must be larger than the start address
- if (in6_addr_cmp(&network->first_address, address) > 0)
-- return 1;
-+ return 0;
-
- // Address must be smaller than the last address
- if (in6_addr_cmp(&network->last_address, address) < 0)
-- return 1;
-+ return 0;
-
- // The address is inside this network
-- return 0;
-+ return 1;
- }
-
- LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) {
-@@ -464,16 +464,18 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth
- }
-
- LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
-- if (loc_network_match_address(self, &other->first_address) == 0)
-+ // Either of the start addresses must be in the other subnet
-+ if (loc_network_match_address(self, &other->first_address))
- return 1;
-
-- if (loc_network_match_address(self, &other->last_address) == 0)
-+ if (loc_network_match_address(other, &self->first_address))
- return 1;
-
-- if (loc_network_match_address(other, &self->first_address) == 0)
-+ // Or either of the end addresses is in the other subnet
-+ if (loc_network_match_address(self, &other->last_address))
- return 1;
-
-- if (loc_network_match_address(other, &self->last_address) == 0)
-+ if (loc_network_match_address(other, &self->last_address))
- return 1;
-
- return 0;
++ if (excluded)
++ loc_network_list_unref(excluded);
++
++ loc_network_list_unref(subnets);
++ loc_network_unref(network1);
++ loc_network_unref(subnet1);
++ loc_network_unref(subnet2);
++ loc_unref(ctx);
++
++ return EXIT_SUCCESS;
++}
diff --git a/src/test-network.c b/src/test-network.c
-index f4cf97b..339743d 100644
+index d38f13d..dde13f1 100644
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -14,6 +14,7 @@
#include <errno.h>
#include <stdio.h>
#include <stddef.h>
-@@ -46,6 +47,13 @@ int main(int argc, char** argv) {
- }
- #endif
+@@ -37,12 +38,21 @@ int main(int argc, char** argv) {
+ // Enable debug logging
+ loc_set_log_priority(ctx, LOG_DEBUG);
++#if 0
+ struct loc_network_tree* tree;
+ err = loc_network_tree_new(ctx, &tree);
+ if (err) {
+ fprintf(stderr, "Could not create the network tree\n");
+ exit(EXIT_FAILURE);
+ }
++#endif
++
+ struct in6_addr address;
+ err = inet_pton(AF_INET6, "2001:db8::1", &address);
+ if (err != 1) {
+ fprintf(stderr, "Could not parse IP address\n");
+ exit(EXIT_FAILURE);
+ }
-+
+
// Create a network
struct loc_network* network1;
- err = loc_network_new_from_string(ctx, &network1, "2001:db8::1/32");
-@@ -92,6 +100,12 @@ int main(int argc, char** argv) {
+@@ -58,12 +68,14 @@ int main(int argc, char** argv) {
exit(EXIT_FAILURE);
}
-+ err = loc_network_match_address(network1, &address);
-+ if (!err) {
-+ fprintf(stderr, "Network1 does not match address\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
- struct loc_network* network2;
- err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48");
++#if 0
+ // Adding network to the tree
+ err = loc_network_tree_add_network(tree, network1);
if (err) {
---
-2.20.1
-
-From 06177d8c6004bf8b54322d92926152a7656f9c2a Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 24 Nov 2020 16:57:28 +0000
-Subject: [PATCH 096/111] network-list: Use binary search to find if a network
- is a subnet
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 13 ++++++-------
- 1 file changed, 6 insertions(+), 7 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index febab95..394bafc 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -697,19 +697,18 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
- while (!loc_network_list_empty(to_check)) {
- struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
-
-+ // Check whether the subnet to check is part of the input list
-+ if (loc_network_list_contains(list, subnet_to_check)) {
-+ loc_network_unref(subnet_to_check);
-+ continue;
-+ }
-+
- // Marks whether this subnet passed all checks
- int passed = 1;
-
- for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
- subnet = loc_network_list_get(list, i);
-
-- // Drop this subnet if is is already in list
-- if (loc_network_cmp(subnet_to_check, subnet) == 0) {
-- passed = 0;
-- loc_network_unref(subnet);
-- break;
-- }
--
- // Drop this subnet if is a subnet of another subnet
- if (loc_network_is_subnet(subnet_to_check, subnet)) {
- passed = 0;
---
-2.20.1
-
-From 77e6d5379ea0de3264be5b8918d620d16e6bcbd3 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Tue, 24 Nov 2020 19:39:35 +0000
-Subject: [PATCH 097/111] network-list: Check last element before doing binary
- search
-
-This is helpful because very often we walk through a list in
-order and are most interested in the last element.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network-list.c | 24 ++++++++++++++++++++++--
- 1 file changed, 22 insertions(+), 2 deletions(-)
-
-diff --git a/src/network-list.c b/src/network-list.c
-index 7e8b5f3..c86ad07 100644
---- a/src/network-list.c
-+++ b/src/network-list.c
-@@ -137,8 +137,26 @@ static off_t loc_network_list_find(struct loc_network_list* list,
- struct loc_network* network, int* found) {
- off_t lo = 0;
- off_t hi = list->size - 1;
-+ int result;
-
-- *found = 0;
-+ // Since we are working on an ordered list, there is often a good chance that
-+ // the network we are looking for is at the end or has to go to the end.
-+ if (hi >= 0) {
-+ result = loc_network_cmp(network, list->elements[hi]);
-+
-+ // Match, so we are done
-+ if (result == 0) {
-+ *found = 1;
-+
-+ return hi;
-+
-+ // This needs to be added after the last one
-+ } else if (result > 0) {
-+ *found = 0;
-+
-+ return hi + 1;
-+ }
-+ }
-
- #ifdef ENABLE_DEBUG
- // Save start time
-@@ -151,7 +169,7 @@ static off_t loc_network_list_find(struct loc_network_list* list,
- i = (lo + hi) / 2;
-
- // Check if this is a match
-- int result = loc_network_cmp(network, list->elements[i]);
-+ result = loc_network_cmp(network, list->elements[i]);
-
- if (result == 0) {
- *found = 1;
-@@ -173,6 +191,8 @@ static off_t loc_network_list_find(struct loc_network_list* list,
- hi = i - 1;
+ fprintf(stderr, "Could not add network to the tree\n");
+ exit(EXIT_FAILURE);
}
++#endif
-+ *found = 0;
-+
- #ifdef ENABLE_DEBUG
- clock_t end = clock();
-
---
-2.20.1
-
-From d42f34ce9b0aa2382ae9b852882c62b34d367cfe Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 14:41:39 +0000
-Subject: [PATCH 098/111] network-list: Set elements pointer to NULL so that we
- know it is empty
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network-list.c | 1 +
- 1 file changed, 1 insertion(+)
-
-diff --git a/src/network-list.c b/src/network-list.c
-index c86ad07..2c4edb3 100644
---- a/src/network-list.c
-+++ b/src/network-list.c
-@@ -104,6 +104,7 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
- loc_network_unref(list->elements[i]);
-
- free(list->elements);
-+ list->elements = NULL;
- list->elements_size = 0;
-
- list->size = 0;
---
-2.20.1
-
-From 673e03f7bfc807248a769cb00ec80f0fa2af7a25 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 14:42:26 +0000
-Subject: [PATCH 099/111] network-list: Do not half list when popping the first
- element
-
-The list was unfortunately halved in size every time an element
-was taken from it, which was great for performance, but shortened
-the result substantially.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network-list.c | 5 ++++-
- 1 file changed, 4 insertions(+), 1 deletion(-)
-
-diff --git a/src/network-list.c b/src/network-list.c
-index 2c4edb3..f4a9d05 100644
---- a/src/network-list.c
-+++ b/src/network-list.c
-@@ -261,10 +261,13 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
- struct loc_network* network = list->elements[0];
-
- // Move all elements to the top of the stack
-- for (unsigned int i = 0; i < --list->size; i++) {
-+ for (unsigned int i = 0; i < list->size - 1; i++) {
- list->elements[i] = list->elements[i+1];
+ // Check if the first and last addresses are correct
+ char* string = loc_network_format_first_address(network1);
+@@ -88,6 +100,12 @@ int main(int argc, char** argv) {
+ exit(EXIT_FAILURE);
}
-+ // The list is shorter now
-+ --list->size;
++ err = loc_network_match_address(network1, &address);
++ if (!err) {
++ fprintf(stderr, "Network1 does not match address\n");
++ exit(EXIT_FAILURE);
++ }
+
- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
-
- return network;
---
-2.20.1
-
-From 9446c7538ab8b27486d74f6dc0dac8bb5c1bbe92 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 14:43:58 +0000
-Subject: [PATCH 100/111] network-list: Show index when listing networks
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network-list.c | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/src/network-list.c b/src/network-list.c
-index f4a9d05..cf9459d 100644
---- a/src/network-list.c
-+++ b/src/network-list.c
-@@ -119,7 +119,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
+ struct loc_network* network2;
+ err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48");
+ if (err) {
+@@ -101,6 +119,7 @@ int main(int argc, char** argv) {
+ exit(EXIT_FAILURE);
+ }
- s = loc_network_str(network);
++#if 0
+ // Adding network to the tree
+ err = loc_network_tree_add_network(tree, network2);
+ if (err) {
+@@ -117,20 +136,84 @@ int main(int argc, char** argv) {
-- INFO(list->ctx, "%s\n", s);
-+ INFO(list->ctx, "%4d: %s\n", i, s);
- free(s);
+ size_t nodes = loc_network_tree_count_nodes(tree);
+ printf("The tree has %zu nodes\n", nodes);
++#endif
++
++ // Check equals function
++ err = loc_network_cmp(network1, network1);
++ if (err) {
++ fprintf(stderr, "Network is not equal with itself\n");
++ exit(EXIT_FAILURE);
++ }
++
++ err = loc_network_cmp(network1, network2);
++ if (!err) {
++ fprintf(stderr, "Networks equal unexpectedly\n");
++ exit(EXIT_FAILURE);
++ }
+
+ // Check subnet function
+- err = loc_network_is_subnet_of(network1, network2);
+- if (err != 0) {
++ err = loc_network_is_subnet(network1, network2);
++ if (!err) {
+ fprintf(stderr, "Subnet check 1 failed: %d\n", err);
+ exit(EXIT_FAILURE);
}
- }
---
-2.20.1
-
-From dc31666be416dbb3bd2bc04127104a3f75b42d5c Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 14:44:23 +0000
-Subject: [PATCH 101/111] network-list: Remove useless comment
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network-list.c | 2 --
- 1 file changed, 2 deletions(-)
-
-diff --git a/src/network-list.c b/src/network-list.c
-index cf9459d..bd3d64e 100644
---- a/src/network-list.c
-+++ b/src/network-list.c
-@@ -132,8 +132,6 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
- return loc_network_ref(list->elements[index]);
- }
--//MOVE FUNCTION GOES HERE
--
- static off_t loc_network_list_find(struct loc_network_list* list,
- struct loc_network* network, int* found) {
- off_t lo = 0;
---
-2.20.1
-
-From 82fa4c92c88cc172953198637bccc375c4d25d20 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 14:44:56 +0000
-Subject: [PATCH 102/111] networks: Add tests for overlaps function
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/test-network.c | 10 ++++++++++
- 1 file changed, 10 insertions(+)
-
-diff --git a/src/test-network.c b/src/test-network.c
-index 339743d..dde13f1 100644
---- a/src/test-network.c
-+++ b/src/test-network.c
-@@ -192,6 +192,16 @@ int main(int argc, char** argv) {
+- err = loc_network_is_subnet_of(network2, network1);
+- if (err != 1) {
++ err = loc_network_is_subnet(network2, network1);
++ if (err) {
+ fprintf(stderr, "Subnet check 2 failed: %d\n", err);
exit(EXIT_FAILURE);
}
++ // Make subnets
++ struct loc_network* subnet1 = NULL;
++ struct loc_network* subnet2 = NULL;
++
++ err = loc_network_subnets(network1, &subnet1, &subnet2);
++ if (err || !subnet1 || !subnet2) {
++ fprintf(stderr, "Could not find subnets of network: %d\n", err);
++ exit(EXIT_FAILURE);
++ }
++
++ char* s = loc_network_str(subnet1);
++ printf("Received subnet1 = %s\n", s);
++ free(s);
++
++ s = loc_network_str(subnet2);
++ printf("Received subnet2 = %s\n", s);
++ free(s);
++
++ if (!loc_network_is_subnet(network1, subnet1)) {
++ fprintf(stderr, "Subnet1 is not a subnet\n");
++ exit(EXIT_FAILURE);
++ }
++
++ if (!loc_network_is_subnet(network1, subnet2)) {
++ fprintf(stderr, "Subnet2 is not a subnet\n");
++ exit(EXIT_FAILURE);
++ }
++
+ if (!loc_network_overlaps(network1, subnet1)) {
+ fprintf(stderr, "Network1 does not seem to contain subnet1\n");
+ exit(EXIT_FAILURE);
+ exit(EXIT_FAILURE);
+ }
+
- loc_network_unref(subnet1);
- loc_network_unref(subnet2);
-
---
-2.20.1
-
-From 0a39d5499ae0c343a6600ea30de7d95e384ef911 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 15:11:21 +0000
-Subject: [PATCH 103/111] networks: Remove comparing family
-
-Everything is encoded in IPv6 anyways...
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 17 -----------------
- 1 file changed, 17 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index 394bafc..66b460f 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -442,12 +442,6 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
- }
-
- LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
-- // Compare family
-- if (self->family > other->family)
-- return 1;
-- else if (self->family < other->family)
-- return -1;
--
- // Compare address
- int r = in6_addr_cmp(&self->first_address, &other->first_address);
- if (r)
-@@ -482,10 +476,6 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
- }
-
- LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
-- // Check family
-- if (self->family != other->family)
-- return 0;
--
- // The prefix must be smaller (this avoids the more complex comparisons later)
- if (self->prefix > other->prefix)
- return 0;
-@@ -601,13 +591,6 @@ ERROR:
-
- static int __loc_network_exclude_to_list(struct loc_network* self,
- struct loc_network* other, struct loc_network_list* list) {
-- // Family must match
-- if (self->family != other->family) {
-- DEBUG(self->ctx, "Family mismatch\n");
--
-- return 1;
-- }
--
- // Other must be a subnet of self
- if (!loc_network_is_subnet(self, other)) {
- DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
---
-2.20.1
-
-From abf559267696061a0c9723d8f8e09c9d1aa8fb75 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 15:13:08 +0000
-Subject: [PATCH 104/111] network: Do not execute with an error when the
- excluded result will be empty
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 6 ++++--
- 1 file changed, 4 insertions(+), 2 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index 66b460f..9aa802f 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -595,14 +595,16 @@ static int __loc_network_exclude_to_list(struct loc_network* self,
- if (!loc_network_is_subnet(self, other)) {
- DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
-
-- return 1;
-+ // Exit silently
-+ return 0;
- }
-
- // We cannot perform this operation if both networks equal
- if (loc_network_cmp(self, other) == 0) {
- DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
-
-- return 1;
-+ // Exit silently
-+ return 0;
- }
-
- return __loc_network_exclude(self, other, list);
---
-2.20.1
-
-From 4fc034e20fdc36267f6ba18e27776fed747fe983 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 15:14:31 +0000
-Subject: [PATCH 105/111] network: Add more excluded networks straight to the
- to_check list
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 6 +-----
- 1 file changed, 1 insertion(+), 5 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index 9aa802f..19c387d 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -705,11 +705,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
- if (loc_network_overlaps(subnet_to_check, subnet)) {
- passed = 0;
-
-- struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet);
-- if (excluded) {
-- loc_network_list_merge(to_check, excluded);
-- loc_network_list_unref(excluded);
-- }
-+ __loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
-
- loc_network_unref(subnet);
- break;
---
-2.20.1
-
-From 4de8ff8e5435225d3375a168054490d2a66b1baf Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 15:15:33 +0000
-Subject: [PATCH 106/111] network: Call subnet function with the correct order
- of arguments
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 4 ++--
- 1 file changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index 19c387d..a96ce6d 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -695,14 +695,14 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
- subnet = loc_network_list_get(list, i);
-
- // Drop this subnet if is a subnet of another subnet
-- if (loc_network_is_subnet(subnet_to_check, subnet)) {
-+ if (loc_network_is_subnet(subnet, subnet_to_check)) {
- passed = 0;
- loc_network_unref(subnet);
- break;
- }
-
- // Break it down if it overlaps
-- if (loc_network_overlaps(subnet_to_check, subnet)) {
-+ if (loc_network_overlaps(subnet, subnet_to_check)) {
- passed = 0;
-
- __loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
---
-2.20.1
-
-From 058e7800e56150bcf0fb8dceaae6fbab1ed239e4 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 15:16:06 +0000
-Subject: [PATCH 107/111] network: Massively improve performance on exclude
-
-When we check the result for any overlaps, we can cut this short
-by walking through both lists from start to end and remember the
-last network that we checked.
-
-The next one will by definition be strictly greater and therefore
-we do not need to check anything before this any more.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 19 +++++++++++++++++--
- 1 file changed, 17 insertions(+), 2 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index a96ce6d..10fa997 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -679,8 +679,10 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
- return NULL;
- }
-
-+ off_t smallest_subnet = 0;
++ loc_network_unref(subnet1);
++ loc_network_unref(subnet2);
+
- while (!loc_network_list_empty(to_check)) {
-- struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
-+ struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check);
-
- // Check whether the subnet to check is part of the input list
- if (loc_network_list_contains(list, subnet_to_check)) {
-@@ -691,7 +693,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
- // Marks whether this subnet passed all checks
- int passed = 1;
-
-- for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
-+ for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) {
- subnet = loc_network_list_get(list, i);
-
- // Drop this subnet if is a subnet of another subnet
-@@ -711,6 +713,19 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
- break;
- }
-
-+ // If the subnet is strictly greater, we do not need to continue the search
-+ r = loc_network_cmp(subnet, subnet_to_check);
-+ if (r > 0) {
-+ loc_network_unref(subnet);
-+ break;
++ struct loc_network_list* excluded = loc_network_exclude(network1, network2);
++ if (!excluded) {
++ fprintf(stderr, "Could not create excluded list\n");
++ exit(EXIT_FAILURE);
++ }
+
-+ // If it is strictly smaller, we can continue the search from here next
-+ // time because all networks that are to be checked can only be larger
-+ // than this one.
-+ } else if (r < 0) {
-+ smallest_subnet = i;
-+ }
++ loc_network_list_dump(excluded);
++ loc_network_list_unref(excluded);
+
- loc_network_unref(subnet);
- }
-
---
-2.20.1
-
-From 9caf6cf5f0fd5eb56ad1678910dbc8017f48863c Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 15:17:21 +0000
-Subject: [PATCH 108/111] network: Remove deprecated sort call
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network.c | 3 ---
- 1 file changed, 3 deletions(-)
-
-diff --git a/src/network.c b/src/network.c
-index 10fa997..4d7ac79 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -738,9 +738,6 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
-
- loc_network_list_unref(to_check);
+ // Create a database
+ struct loc_writer* writer;
+ err = loc_writer_new(ctx, &writer, NULL, NULL);
+@@ -160,6 +243,28 @@ int main(int argc, char** argv) {
+ // Set ASN
+ loc_network_set_asn(network4, 1024);
-- // Sort the result
-- loc_network_list_sort(subnets);
--
- return subnets;
- }
++ // Try adding an invalid network
++ struct loc_network* network;
++ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
++ if (err != -EINVAL) {
++ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
++ exit(EXIT_FAILURE);
++ }
++
++ // Try adding a single address
++ err = loc_writer_add_network(writer, &network, "2001:db8::");
++ if (err) {
++ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
++ exit(EXIT_FAILURE);
++ }
++
++ // Try adding localhost
++ err = loc_writer_add_network(writer, &network, "::1/128");
++ if (err != -EINVAL) {
++ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
++ exit(EXIT_FAILURE);
++ }
++
+ FILE* f = tmpfile();
+ if (!f) {
+ fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
+@@ -177,7 +282,10 @@ int main(int argc, char** argv) {
+ loc_network_unref(network2);
+ loc_network_unref(network3);
+ loc_network_unref(network4);
++
++#if 0
+ loc_network_tree_unref(tree);
++#endif
---
-2.20.1
-
-From 8ebf1d3912020d5c0ed98bb32a0640952bd6447c Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 15:17:42 +0000
-Subject: [PATCH 109/111] network-list: Include network header
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/loc/network-list.h | 2 ++
- 1 file changed, 2 insertions(+)
-
-diff --git a/src/loc/network-list.h b/src/loc/network-list.h
-index 21c7402..bee21c4 100644
---- a/src/loc/network-list.h
-+++ b/src/loc/network-list.h
-@@ -17,6 +17,8 @@
- #ifndef LIBLOC_NETWORK_LIST_H
- #define LIBLOC_NETWORK_LIST_H
+ // And open it again from disk
+ struct loc_database* db;
+diff --git a/src/writer.c b/src/writer.c
+index 5939cff..2f09b56 100644
+--- a/src/writer.c
++++ b/src/writer.c
+@@ -147,8 +147,19 @@ static void loc_writer_free(struct loc_writer* writer) {
+ EVP_PKEY_free(writer->private_key2);
-+#include <loc/network.h>
+ // Unref all AS
+- for (unsigned int i = 0; i < writer->as_count; i++) {
+- loc_as_unref(writer->as[i]);
++ if (writer->as) {
++ for (unsigned int i = 0; i < writer->as_count; i++) {
++ loc_as_unref(writer->as[i]);
++ }
++ free(writer->as);
++ }
+
- struct loc_network_list;
- int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
- struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
---
-2.20.1
-
-From fff093b6827a31ff84a6d5150bfede140b696d25 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 15:24:43 +0000
-Subject: [PATCH 110/111] network-list: Use clear function to tidy up
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- src/network-list.c | 4 ++--
- 1 file changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/src/network-list.c b/src/network-list.c
-index bd3d64e..f3458b4 100644
---- a/src/network-list.c
-+++ b/src/network-list.c
-@@ -70,8 +70,8 @@ LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list
- static void loc_network_list_free(struct loc_network_list* list) {
- DEBUG(list->ctx, "Releasing network list at %p\n", list);
-
-- for (unsigned int i = 0; i < list->size; i++)
-- loc_network_unref(list->elements[i]);
-+ // Remove all content
-+ loc_network_list_clear(list);
++ // Unref all countries
++ if (writer->countries) {
++ for (unsigned int i = 0; i < writer->countries_count; i++) {
++ loc_country_unref(writer->countries[i]);
++ }
++ free(writer->countries);
+ }
- loc_unref(list->ctx);
- free(list);
---
-2.20.1
-
-From adbec5c0317a1c007a897fe3f63f0f86ff448124 Mon Sep 17 00:00:00 2001
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed, 25 Nov 2020 20:00:46 +0000
-Subject: [PATCH 111/111] configure: Bump version to 0.9.5
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- configure.ac | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/configure.ac b/configure.ac
-index 2364dfd..012d8ca 100644
---- a/configure.ac
-+++ b/configure.ac
-@@ -1,6 +1,6 @@
- AC_PREREQ(2.60)
- AC_INIT([libloc],
-- [0.9.4],
-+ [0.9.5],
- [location@lists.ipfire.org],
- [libloc],
- [https://location.ipfire.org/])
---
-2.20.1
-
+ // Release network tree
projects / people / pmueller / ipfire-2.x.git / blobdiff