Fix merging the database with SQLite

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/tools/base.py b/tools/base.py
index e52287549beb499eca7bbcb1cacd5b6170a239fa..b57f485ee7a84e1d5c72e3229ae771f892340338 100644 (file)
--- a/tools/base.py
+++ b/tools/base.py
@@ -103,13 +103,17 @@ class RIRParser(object):
                        cursor.executescript("""
                                CREATE TABLE IF NOT EXISTS autnums(asn INTEGER, name TEXT, org TEXT);
 
-                               CREATE TABLE IF NOT EXISTS inetnums(network TEXT, netname TEXT, country TEXT, description TEXT);
+                               CREATE TABLE IF NOT EXISTS inetnums(network TEXT, netname TEXT, country TEXT, description TEXT,
+                                       family INTEGER, address_start0 INTEGER, address_start1 INTEGER,
+                                       address_end0 INTEGER, address_end1 INTEGER, prefix INTEGER);
+                               CREATE INDEX inetnums_search ON inetnums(family, prefix, address_start0, address_start1);
 
                                CREATE TABLE IF NOT EXISTS organisations(handle TEXT, name TEXT, country TEXT);
                                CREATE INDEX IF NOT EXISTS organisations_handle ON organisations(handle);
 
-                               CREATE TABLE IF NOT EXISTS routes(route TEXT, asn INTEGER);
-                               CREATE INDEX IF NOT EXISTS routes_route ON routes(route);
+                               CREATE TABLE IF NOT EXISTS routes(route TEXT, asn INTEGER,
+                                       family INTEGER, address_start0 INTEGER, address_start1 INTEGER,
+                                       address_end0 INTEGER, address_end1 INTEGER, prefix INTEGER);
                        """)
 
                return db
@@ -154,10 +158,41 @@ class RIRParser(object):
 
                with self.db as c:
                        # Write all networks
-                       res = c.execute("""SELECT inetnums.network, routes.asn,
-                               inetnums.country, inetnums.netname, inetnums.description
-                               FROM inetnums LEFT JOIN routes ON inetnums.network = routes.route
-                               ORDER BY routes.asn, inetnums.network""")
+                       res = c.execute("""
+                               SELECT networks.route,
+                                       networks.asn,
+                                       networks.country,
+                                       networks.netname,
+                                       networks.description FROM (
+                                               SELECT DISTINCT routes.route,
+                                                       routes.address_start0,
+                                                       routes.address_start1,
+                                                       inetnums.network,
+                                                       routes.asn,
+                                                       inetnums.country,
+                                                       inetnums.netname,
+                                                       inetnums.description
+                                               FROM routes
+                                               LEFT JOIN inetnums
+                                                       WHERE routes.family = inetnums.family
+                                                       AND routes.prefix >= inetnums.prefix
+                                                       AND (
+                                                               inetnums.address_start0 < routes.address_start0
+                                                               OR (
+                                                                       inetnums.address_start0 = routes.address_start0
+                                                                       AND inetnums.address_start1 <= routes.address_start1
+                                                               )
+                                                       ) AND (
+                                                               inetnums.address_end0 > routes.address_end0
+                                                               OR (
+                                                                       inetnums.address_end0 = routes.address_end0
+                                                                       AND inetnums.address_end1 >= routes.address_end1
+                                                               )
+                                                       )
+                                               ORDER BY inetnums.prefix DESC
+                                       ) networks
+                               ORDER BY networks.address_start0, networks.address_start1
+                       """)
 
                        for row in res:
                                net, asn, country, name, description = row
@@ -328,15 +363,27 @@ class RIRParser(object):
                        return
 
                with self.db as c:
+                       network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
+                       # Get the first and last address of this network
+                       address_start, address_end = network.network_address, network.broadcast_address
+
                        args = (
-                               inetnum.get("inet6num") or inetnum.get("inetnum"),
+                               "%s" % network,
                                inetnum.get("netname"),
                                inetnum.get("country"),
                                inetnum.get("descr"),
+                               network.version,
+                               int(address_start) >> 64,
+                               int(address_start) & 0xffffffff,
+                               int(address_end) >> 64,
+                               int(address_end) & 0xffffffff,
+                               network.prefixlen,
                        )
 
-                       c.execute("INSERT INTO inetnums(network, netname, country, description) \
-                               VALUES(?, ?, ?, ?)", args)
+                       c.execute("INSERT INTO inetnums(network, netname, country, description, family, \
+                               address_start0, address_start1, address_end0, address_end1, prefix) \
+                               VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", args)
 
        def _parse_route_block(self, block):
                logging.debug("Parsing route block:")
@@ -362,13 +409,25 @@ class RIRParser(object):
                        return
 
                with self.db as c:
+                       network = ipaddress.ip_network(route.get("route6") or route.get("route"), strict=False)
+
+                       # Get the first and last address of this network
+                       address_start, address_end = network.network_address, network.broadcast_address
+
                        args = (
-                               route.get("route6") or route.get("route"),
+                               "%s" % network,
                                route.get("asn"),
+                               network.version,
+                               int(address_start) >> 64,
+                               int(address_start) & 0xffffffff,
+                               int(address_end) >> 64,
+                               int(address_end) & 0xffffffff,
+                               network.prefixlen,
                        )
 
-                       c.execute("INSERT INTO routes(route, asn) \
-                               VALUES(?, ?)", args)
+                       c.execute("INSERT INTO routes(route, asn, family, \
+                               address_start0, address_start1, address_end0, address_end1, prefix) \
+                               VALUES(?, ?, ?, ?, ?, ?, ?, ?)", args)
 
        def _parse_autnum_block(self, block):
                logging.debug("Parsing autnum block:")