export: Conditionally enable flattening

By default, we enabled flattening of the network tree when we export it.

However, this is only required for xt_geoip since the other formats can
deal with overlapping networks and would even benefit from a shorter
list.

Therefore this is now only enabled when needed which results in shorter
export times (9 seconds instead of 2.5 minutes) and the full ipset is
about 20% smaller when loaded into memory than before.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/python/export.py b/src/python/export.py
index 10f84f45f3ba6abd03b32e107f8d3f986079ac4d..c5d7475408c18474e58d065691642eabeed413ec 100644 (file)
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -41,6 +41,9 @@ class OutputWriter(object):
        suffix = "networks"
        mode = "w"
 
+       # Enable network flattening (i.e. networks cannot overlap)
+       flatten = False
+
        def __init__(self, f, family=None, prefix=None):
                self.f = f
                self.prefix = prefix
@@ -177,6 +180,7 @@ class XTGeoIPOutputWriter(OutputWriter):
        """
        suffix = "iv"
        mode = "wb"
+       flatten = True
 
        def write(self, network):
                self.f.write(network._first_address)
@@ -223,7 +227,7 @@ class Exporter(object):
 
                        # Get all networks that match the family
                        networks = self.db.search_networks(family=family,
-                               country_codes=country_codes, asns=asns, flatten=True)
+                               country_codes=country_codes, asns=asns, flatten=self.writer.flatten)
 
                        # Walk through all networks
                        for network in networks: