export: Skip writing any subnets

It is unnecessary to write any subnets in the output, because
they might already be included in a bigger parent network.

Since the tree is ordered, we only need to check if the network
to be written is a subnet of the previous one. If so, we skip it.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/libloc.sym b/src/libloc.sym
index 9a1e6f0601d05f553bfa62127b24701ed3397a50..222a4cf7ef6b9e0008b18ad74af2ec1d0a0db3a7 100644 (file)
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -83,6 +83,7 @@ global:
        loc_network_get_asn;
        loc_network_get_country_code;
        loc_network_has_flag;
+       loc_network_is_subnet_of;
        loc_network_match_asn;
        loc_network_match_country_code;
        loc_network_match_flag;

diff --git a/src/loc/network.h b/src/loc/network.h
index 273041c8738ceda1ddb85ca29e64e6f1613510d7..c26092f8d8c3ee7cdd1446a92c3bb577d868ef48 100644 (file)
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -51,6 +51,8 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
 
+int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
+
 #ifdef LIBLOC_PRIVATE
 
 int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);

diff --git a/src/network.c b/src/network.c
index 5a8db7a58b7f30e2286bb23653cd7c254d13ff2b..286e13685498befb84c80db7421cf47ff450ad9b 100644 (file)
--- a/src/network.c
+++ b/src/network.c
@@ -346,6 +346,24 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
        return loc_network_has_flag(network, flag);
 }
 
+LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
+       // If the start address of the other network is smaller than this network,
+       // it cannot be a subnet.
+       if (in6_addr_cmp(&self->start_address, &other->start_address) < 0)
+               return 0;
+
+       // Get the end addresses
+       struct in6_addr last_address_self  = make_last_address(&self->start_address,  self->prefix);
+       struct in6_addr last_address_other = make_last_address(&other->start_address, other->prefix);
+
+       // If the end address of the other network is greater than this network,
+       // it cannot be a subnet.
+       if (in6_addr_cmp(&last_address_self, &last_address_other) > 0)
+               return 0;
+
+       return 1;
+}
+
 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
        // Add country code
        loc_country_code_copy(dbobj->country_code, network->country_code);

diff --git a/src/python/export.py b/src/python/export.py
index d0bbe77e9796a61a0bb38b3e063518e9f0cfba8e..180d462fcad0e7e075afdb2082ca51a61152bfca 100644 (file)
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -39,8 +39,11 @@ class OutputWriter(object):
        suffix = "networks"
        mode = "w"
 
-       def __init__(self, f, prefix=None):
-               self.f, self.prefix = f, prefix
+       def __init__(self, f, prefix=None, flatten=True):
+               self.f, self.prefix, self.flatten = f, prefix, flatten
+
+               # The previously written network
+               self._last_network = None
 
                # Immediately write the header
                self._write_header()
@@ -57,6 +60,18 @@ class OutputWriter(object):
        def __repr__(self):
                return "<%s f=%s>" % (self.__class__.__name__, self.f)
 
+       def _flatten(self, network):
+               """
+                       Checks if the given network needs to be written to file,
+                       or if it is a subnet of the previously written network.
+               """
+               if self._last_network and network.is_subnet_of(self._last_network):
+                       return True
+
+               # Remember this network for the next call
+               self._last_network = network
+               return False
+
        def _write_header(self):
                """
                        The header of the file
@@ -69,9 +84,17 @@ class OutputWriter(object):
                """
                pass
 
-       def write(self, network):
+       def _write_network(self, network):
                self.f.write("%s\n" % network)
 
+       def write(self, network):
+               if self.flatten and self._flatten(network):
+                       log.debug("Skipping writing network %s" % network)
+                       return
+
+               # Write the network to file
+               self._write_network(network)
+
        def finish(self):
                """
                        Called when all data has been written
@@ -91,7 +114,7 @@ class IpsetOutputWriter(OutputWriter):
        def _write_header(self):
                self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
 
-       def write(self, network):
+       def _write_network(self, network):
                self.f.write("add %s %s\n" % (self.prefix, network))
 
 
@@ -107,7 +130,7 @@ class NftablesOutputWriter(OutputWriter):
        def _write_footer(self):
                self.f.write("}\n")
 
-       def write(self, network):
+       def _write_network(self, network):
                self.f.write("  %s,\n" % network)
 
 
@@ -119,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter):
        suffix = "iv"
        mode = "wb"
 
-       def write(self, network):
+       def _write_network(self, network):
                n = ipaddress.ip_network("%s" % network)
 
                for address in (n.network_address, n.broadcast_address):

diff --git a/src/python/network.c b/src/python/network.c
index d1b0de8992dfb6a17d00240cc7ec8e9e10e0e6d0..5fbb17d2d2cf79303319d6465d87bc55e63b5b69 100644 (file)
--- a/src/python/network.c
+++ b/src/python/network.c
@@ -154,6 +154,18 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
        Py_RETURN_NONE;
 }
 
+static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
+       NetworkObject* other = NULL;
+
+       if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
+               return NULL;
+
+       if (loc_network_is_subnet_of(self->network, other->network))
+               Py_RETURN_TRUE;
+
+       Py_RETURN_FALSE;
+}
+
 static struct PyMethodDef Network_methods[] = {
        {
                "has_flag",
@@ -161,6 +173,12 @@ static struct PyMethodDef Network_methods[] = {
                METH_VARARGS,
                NULL,
        },
+       {
+               "is_subnet_of",
+               (PyCFunction)Network_is_subnet_of,
+               METH_VARARGS,
+               NULL,
+       },
        {
                "set_flag",
                (PyCFunction)Network_set_flag,

diff --git a/src/test-network.c b/src/test-network.c
index 364099d51d8fc2064f962f8f5e849a2fd067b8c3..c49c3ef4038a23c56bb75d136d275ced3b40fbc9 100644 (file)
--- a/src/test-network.c
+++ b/src/test-network.c
@@ -95,6 +95,19 @@ int main(int argc, char** argv) {
        size_t nodes = loc_network_tree_count_nodes(tree);
        printf("The tree has %zu nodes\n", nodes);
 
+       // Check subnet function
+       err = loc_network_is_subnet_of(network1, network2);
+       if (err != 0) {
+               fprintf(stderr, "Subnet check 1 failed: %d\n", err);
+               exit(EXIT_FAILURE);
+       }
+
+       err = loc_network_is_subnet_of(network2, network1);
+       if (err != 1) {
+               fprintf(stderr, "Subnet check 2 failed: %d\n", err);
+               exit(EXIT_FAILURE);
+       }
+
        // Create a database
        struct loc_writer* writer;
        err = loc_writer_new(ctx, &writer, NULL, NULL);