Revert "Replace sqlite with PostgreSQL"

This reverts commit 44c781ac664bf0f6670651e0a721fe2ddc446e66.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/tools/base.py b/tools/base.py
index 119723bddfbfc57da3752b6d71b43aa85392ba25..e52287549beb499eca7bbcb1cacd5b6170a239fa 100644 (file)
--- a/tools/base.py
+++ b/tools/base.py
@@ -25,8 +25,8 @@ import ipaddress
 import logging
 import math
 import os.path
-import psycopg2
 import re
+import sqlite3
 
 from . import downloader
 from . import util
@@ -90,38 +90,27 @@ class RIRParser(object):
                self.downloader = downloader.Downloader()
 
                # Create a database to hold temporary data
-               self.db = self._make_database(dbname="location", user="location")
+               self.db = self._make_database(":memory:")
 
                # Start time
                self.start_time = datetime.datetime.utcnow()
 
-       def _make_database(self, **kwargs):
-               db = psycopg2.connect(**kwargs)
+       def _make_database(self, filename):
+               db = sqlite3.connect(filename)
 
                # Create database layout
-               with db.cursor() as cursor:
-                       # Autnums
-                       cursor.execute("CREATE TABLE IF NOT EXISTS autnums(asn integer, name text, org text)")
-                       cursor.execute("TRUNCATE TABLE autnums")
-
-                       # Inetnums
-                       cursor.execute("CREATE TABLE IF NOT EXISTS inetnums(network inet, netname text, \
-                               country text, description text)")
-                       cursor.execute("CREATE INDEX IF NOT EXISTS inetnums_family ON inetnums(family(network))")
-                       cursor.execute("TRUNCATE TABLE inetnums")
-
-                       # Organizations
-                       cursor.execute("CREATE TABLE IF NOT EXISTS organisations(handle text, name text, country text)")
-                       cursor.execute("CREATE INDEX IF NOT EXISTS organisations_handle ON organisations(handle)")
-                       cursor.execute("TRUNCATE TABLE organisations")
-
-                       # Routes
-                       cursor.execute("CREATE TABLE IF NOT EXISTS routes(route inet, asn integer)")
-                       cursor.execute("CREATE INDEX IF NOT EXISTS routes_route ON routes(route)")
-                       cursor.execute("CREATE INDEX IF NOT EXISTS routes_family ON routes(family(route))")
-                       cursor.execute("TRUNCATE TABLE routes")
-
-                       db.commit()
+               with db as cursor:
+                       cursor.executescript("""
+                               CREATE TABLE IF NOT EXISTS autnums(asn INTEGER, name TEXT, org TEXT);
+
+                               CREATE TABLE IF NOT EXISTS inetnums(network TEXT, netname TEXT, country TEXT, description TEXT);
+
+                               CREATE TABLE IF NOT EXISTS organisations(handle TEXT, name TEXT, country TEXT);
+                               CREATE INDEX IF NOT EXISTS organisations_handle ON organisations(handle);
+
+                               CREATE TABLE IF NOT EXISTS routes(route TEXT, asn INTEGER);
+                               CREATE INDEX IF NOT EXISTS routes_route ON routes(route);
+                       """)
 
                return db
 
@@ -138,13 +127,13 @@ class RIRParser(object):
                # Write header
                self._write_header(f)
 
-               with self.db.cursor() as c:
-                       c.execute("""SELECT DISTINCT autnums.asn, autnums.name,
+               with self.db as c:
+                       res = c.execute("""SELECT DISTINCT autnums.asn, autnums.name,
                                organisations.name, organisations.country FROM autnums
                                LEFT JOIN organisations ON autnums.org = organisations.handle
                                WHERE autnums.asn IS NOT NULL ORDER BY autnums.asn""")
 
-                       for row in c:
+                       for row in res:
                                f.write(FMT % ("asnum:", "AS%s" % row[0]))
 
                                if row[1]:
@@ -163,18 +152,14 @@ class RIRParser(object):
                # Write header
                self._write_header(f)
 
-               with self.db.cursor() as c:
+               with self.db as c:
                        # Write all networks
-                       c.execute("""SELECT DISTINCT ON (routes.route)
-                               routes.route, routes.asn, inetnums.country, inetnums.netname, inetnums.description
-                               FROM routes
-                                       LEFT JOIN inetnums
-                                               ON family(routes.route) = family(inetnums.network)
-                                               AND routes.route <<= inetnums.network
-                               ORDER BY routes.route, masklen(routes.route) DESC
-                       """)
+                       res = c.execute("""SELECT inetnums.network, routes.asn,
+                               inetnums.country, inetnums.netname, inetnums.description
+                               FROM inetnums LEFT JOIN routes ON inetnums.network = routes.route
+                               ORDER BY routes.asn, inetnums.network""")
 
-                       for row in c:
+                       for row in res:
                                net, asn, country, name, description = row
 
                                f.write(FMT % ("net:", net))
@@ -342,7 +327,7 @@ class RIRParser(object):
                if not inetnum:
                        return
 
-               with self.db.cursor() as c:
+               with self.db as c:
                        args = (
                                inetnum.get("inet6num") or inetnum.get("inetnum"),
                                inetnum.get("netname"),
@@ -351,7 +336,7 @@ class RIRParser(object):
                        )
 
                        c.execute("INSERT INTO inetnums(network, netname, country, description) \
-                               VALUES(%s, %s, %s, %s)", args)
+                               VALUES(?, ?, ?, ?)", args)
 
        def _parse_route_block(self, block):
                logging.debug("Parsing route block:")
@@ -376,14 +361,14 @@ class RIRParser(object):
                if not route:
                        return
 
-               with self.db.cursor() as c:
+               with self.db as c:
                        args = (
                                route.get("route6") or route.get("route"),
                                route.get("asn"),
                        )
 
                        c.execute("INSERT INTO routes(route, asn) \
-                               VALUES(%s, %s)", args)
+                               VALUES(?, ?)", args)
 
        def _parse_autnum_block(self, block):
                logging.debug("Parsing autnum block:")
@@ -407,7 +392,7 @@ class RIRParser(object):
                if not autnum:
                        return
 
-               with self.db.cursor() as c:
+               with self.db as c:
                        args = (
                                autnum.get("asn"),
                                autnum.get("as-name"),
@@ -415,7 +400,7 @@ class RIRParser(object):
                        )
 
                        c.execute("INSERT INTO autnums(asn, name, org) \
-                               VALUES(%s, %s, %s)", args)
+                               VALUES(?, ?, ?)", args)
        
        def _parse_org_block(self, block):
                logging.debug("Parsing org block:")
@@ -434,7 +419,7 @@ class RIRParser(object):
                if not org:
                        return
 
-               with self.db.cursor() as c:
+               with self.db as c:
                        args = (
                                org.get("organisation"),
                                org.get("org-name"),
@@ -442,4 +427,4 @@ class RIRParser(object):
                        )
 
                        c.execute("INSERT INTO organisations(handle, name, country) \
-                               VALUES(%s, %s, %s)", args)
+                               VALUES(?, ?, ?)", args)