From: Michael Tremer Date: Sun, 13 Jan 2019 15:40:06 +0000 (+0000) Subject: Revert "Replace sqlite with PostgreSQL" X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1ca58c91114ead53ddd97cdbfc8d0e45fac79818;p=location%2Flocation-database.git Revert "Replace sqlite with PostgreSQL" This reverts commit 44c781ac664bf0f6670651e0a721fe2ddc446e66. Signed-off-by: Michael Tremer --- diff --git a/tools/base.py b/tools/base.py index 119723b..e522875 100644 --- a/tools/base.py +++ b/tools/base.py @@ -25,8 +25,8 @@ import ipaddress import logging import math import os.path -import psycopg2 import re +import sqlite3 from . import downloader from . import util @@ -90,38 +90,27 @@ class RIRParser(object): self.downloader = downloader.Downloader() # Create a database to hold temporary data - self.db = self._make_database(dbname="location", user="location") + self.db = self._make_database(":memory:") # Start time self.start_time = datetime.datetime.utcnow() - def _make_database(self, **kwargs): - db = psycopg2.connect(**kwargs) + def _make_database(self, filename): + db = sqlite3.connect(filename) # Create database layout - with db.cursor() as cursor: - # Autnums - cursor.execute("CREATE TABLE IF NOT EXISTS autnums(asn integer, name text, org text)") - cursor.execute("TRUNCATE TABLE autnums") - - # Inetnums - cursor.execute("CREATE TABLE IF NOT EXISTS inetnums(network inet, netname text, \ - country text, description text)") - cursor.execute("CREATE INDEX IF NOT EXISTS inetnums_family ON inetnums(family(network))") - cursor.execute("TRUNCATE TABLE inetnums") - - # Organizations - cursor.execute("CREATE TABLE IF NOT EXISTS organisations(handle text, name text, country text)") - cursor.execute("CREATE INDEX IF NOT EXISTS organisations_handle ON organisations(handle)") - cursor.execute("TRUNCATE TABLE organisations") - - # Routes - cursor.execute("CREATE TABLE IF NOT EXISTS routes(route inet, asn integer)") - cursor.execute("CREATE INDEX IF NOT EXISTS routes_route ON routes(route)") - cursor.execute("CREATE INDEX IF NOT EXISTS routes_family ON routes(family(route))") - cursor.execute("TRUNCATE TABLE routes") - - db.commit() + with db as cursor: + cursor.executescript(""" + CREATE TABLE IF NOT EXISTS autnums(asn INTEGER, name TEXT, org TEXT); + + CREATE TABLE IF NOT EXISTS inetnums(network TEXT, netname TEXT, country TEXT, description TEXT); + + CREATE TABLE IF NOT EXISTS organisations(handle TEXT, name TEXT, country TEXT); + CREATE INDEX IF NOT EXISTS organisations_handle ON organisations(handle); + + CREATE TABLE IF NOT EXISTS routes(route TEXT, asn INTEGER); + CREATE INDEX IF NOT EXISTS routes_route ON routes(route); + """) return db @@ -138,13 +127,13 @@ class RIRParser(object): # Write header self._write_header(f) - with self.db.cursor() as c: - c.execute("""SELECT DISTINCT autnums.asn, autnums.name, + with self.db as c: + res = c.execute("""SELECT DISTINCT autnums.asn, autnums.name, organisations.name, organisations.country FROM autnums LEFT JOIN organisations ON autnums.org = organisations.handle WHERE autnums.asn IS NOT NULL ORDER BY autnums.asn""") - for row in c: + for row in res: f.write(FMT % ("asnum:", "AS%s" % row[0])) if row[1]: @@ -163,18 +152,14 @@ class RIRParser(object): # Write header self._write_header(f) - with self.db.cursor() as c: + with self.db as c: # Write all networks - c.execute("""SELECT DISTINCT ON (routes.route) - routes.route, routes.asn, inetnums.country, inetnums.netname, inetnums.description - FROM routes - LEFT JOIN inetnums - ON family(routes.route) = family(inetnums.network) - AND routes.route <<= inetnums.network - ORDER BY routes.route, masklen(routes.route) DESC - """) + res = c.execute("""SELECT inetnums.network, routes.asn, + inetnums.country, inetnums.netname, inetnums.description + FROM inetnums LEFT JOIN routes ON inetnums.network = routes.route + ORDER BY routes.asn, inetnums.network""") - for row in c: + for row in res: net, asn, country, name, description = row f.write(FMT % ("net:", net)) @@ -342,7 +327,7 @@ class RIRParser(object): if not inetnum: return - with self.db.cursor() as c: + with self.db as c: args = ( inetnum.get("inet6num") or inetnum.get("inetnum"), inetnum.get("netname"), @@ -351,7 +336,7 @@ class RIRParser(object): ) c.execute("INSERT INTO inetnums(network, netname, country, description) \ - VALUES(%s, %s, %s, %s)", args) + VALUES(?, ?, ?, ?)", args) def _parse_route_block(self, block): logging.debug("Parsing route block:") @@ -376,14 +361,14 @@ class RIRParser(object): if not route: return - with self.db.cursor() as c: + with self.db as c: args = ( route.get("route6") or route.get("route"), route.get("asn"), ) c.execute("INSERT INTO routes(route, asn) \ - VALUES(%s, %s)", args) + VALUES(?, ?)", args) def _parse_autnum_block(self, block): logging.debug("Parsing autnum block:") @@ -407,7 +392,7 @@ class RIRParser(object): if not autnum: return - with self.db.cursor() as c: + with self.db as c: args = ( autnum.get("asn"), autnum.get("as-name"), @@ -415,7 +400,7 @@ class RIRParser(object): ) c.execute("INSERT INTO autnums(asn, name, org) \ - VALUES(%s, %s, %s)", args) + VALUES(?, ?, ?)", args) def _parse_org_block(self, block): logging.debug("Parsing org block:") @@ -434,7 +419,7 @@ class RIRParser(object): if not org: return - with self.db.cursor() as c: + with self.db as c: args = ( org.get("organisation"), org.get("org-name"), @@ -442,4 +427,4 @@ class RIRParser(object): ) c.execute("INSERT INTO organisations(handle, name, country) \ - VALUES(%s, %s, %s)", args) + VALUES(?, ?, ?)", args)