From: Michael Tremer <michael.tremer@ipfire.org>
Date: Sun, 13 Jan 2019 22:38:41 +0000 (+0000)
Subject: Fix merging the database with SQLite
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f1c07498b095f0024d2295e96b1733ade674138a;p=location%2Flocation-database.git

Fix merging the database with SQLite

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/tools/base.py b/tools/base.py
index e522875..b57f485 100644
--- a/tools/base.py
+++ b/tools/base.py
@@ -103,13 +103,17 @@ class RIRParser(object):
 			cursor.executescript("""
 				CREATE TABLE IF NOT EXISTS autnums(asn INTEGER, name TEXT, org TEXT);
 
-				CREATE TABLE IF NOT EXISTS inetnums(network TEXT, netname TEXT, country TEXT, description TEXT);
+				CREATE TABLE IF NOT EXISTS inetnums(network TEXT, netname TEXT, country TEXT, description TEXT,
+					family INTEGER, address_start0 INTEGER, address_start1 INTEGER,
+					address_end0 INTEGER, address_end1 INTEGER, prefix INTEGER);
+				CREATE INDEX inetnums_search ON inetnums(family, prefix, address_start0, address_start1);
 
 				CREATE TABLE IF NOT EXISTS organisations(handle TEXT, name TEXT, country TEXT);
 				CREATE INDEX IF NOT EXISTS organisations_handle ON organisations(handle);
 
-				CREATE TABLE IF NOT EXISTS routes(route TEXT, asn INTEGER);
-				CREATE INDEX IF NOT EXISTS routes_route ON routes(route);
+				CREATE TABLE IF NOT EXISTS routes(route TEXT, asn INTEGER,
+					family INTEGER, address_start0 INTEGER, address_start1 INTEGER,
+					address_end0 INTEGER, address_end1 INTEGER, prefix INTEGER);
 			""")
 
 		return db
@@ -154,10 +158,41 @@ class RIRParser(object):
 
 		with self.db as c:
 			# Write all networks
-			res = c.execute("""SELECT inetnums.network, routes.asn,
-				inetnums.country, inetnums.netname, inetnums.description
-				FROM inetnums LEFT JOIN routes ON inetnums.network = routes.route
-				ORDER BY routes.asn, inetnums.network""")
+			res = c.execute("""
+				SELECT networks.route,
+					networks.asn,
+					networks.country,
+					networks.netname,
+					networks.description FROM (
+						SELECT DISTINCT routes.route,
+							routes.address_start0,
+							routes.address_start1,
+							inetnums.network,
+							routes.asn,
+							inetnums.country,
+							inetnums.netname,
+							inetnums.description
+						FROM routes
+						LEFT JOIN inetnums
+							WHERE routes.family = inetnums.family
+							AND routes.prefix >= inetnums.prefix
+							AND (
+								inetnums.address_start0 < routes.address_start0
+								OR (
+									inetnums.address_start0 = routes.address_start0
+									AND inetnums.address_start1 <= routes.address_start1
+								)
+							) AND (
+								inetnums.address_end0 > routes.address_end0
+								OR (
+									inetnums.address_end0 = routes.address_end0
+									AND inetnums.address_end1 >= routes.address_end1
+								)
+							)
+						ORDER BY inetnums.prefix DESC
+					) networks
+				ORDER BY networks.address_start0, networks.address_start1
+			""")
 
 			for row in res:
 				net, asn, country, name, description = row
@@ -328,15 +363,27 @@ class RIRParser(object):
 			return
 
 		with self.db as c:
+			network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
+			# Get the first and last address of this network
+			address_start, address_end = network.network_address, network.broadcast_address
+
 			args = (
-				inetnum.get("inet6num") or inetnum.get("inetnum"),
+				"%s" % network,
 				inetnum.get("netname"),
 				inetnum.get("country"),
 				inetnum.get("descr"),
+				network.version,
+				int(address_start) >> 64,
+				int(address_start) & 0xffffffff,
+				int(address_end) >> 64,
+				int(address_end) & 0xffffffff,
+				network.prefixlen,
 			)
 
-			c.execute("INSERT INTO inetnums(network, netname, country, description) \
-				VALUES(?, ?, ?, ?)", args)
+			c.execute("INSERT INTO inetnums(network, netname, country, description, family, \
+				address_start0, address_start1, address_end0, address_end1, prefix) \
+				VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", args)
 
 	def _parse_route_block(self, block):
 		logging.debug("Parsing route block:")
@@ -362,13 +409,25 @@ class RIRParser(object):
 			return
 
 		with self.db as c:
+			network = ipaddress.ip_network(route.get("route6") or route.get("route"), strict=False)
+
+			# Get the first and last address of this network
+			address_start, address_end = network.network_address, network.broadcast_address
+
 			args = (
-				route.get("route6") or route.get("route"),
+				"%s" % network,
 				route.get("asn"),
+				network.version,
+				int(address_start) >> 64,
+				int(address_start) & 0xffffffff,
+				int(address_end) >> 64,
+				int(address_end) & 0xffffffff,
+				network.prefixlen,
 			)
 
-			c.execute("INSERT INTO routes(route, asn) \
-				VALUES(?, ?)", args)
+			c.execute("INSERT INTO routes(route, asn, family, \
+				address_start0, address_start1, address_end0, address_end1, prefix) \
+				VALUES(?, ?, ?, ?, ?, ?, ?, ?)", args)
 
 	def _parse_autnum_block(self, block):
 		logging.debug("Parsing autnum block:")