From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 30 Jan 2018 16:44:56 +0000 (+0000)
Subject: Add script to export data into a location database
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0c4fda20f039334307ebbe6c54f1155c863da661;p=location%2Flocation-database.git

Add script to export data into a location database

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/compile-database b/compile-database
new file mode 100755
index 0000000..f225186
--- /dev/null
+++ b/compile-database
@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+###############################################################################
+#                                                                             #
+# location-database - A database to determine someone's                       #
+#                     location on the Internet                                #
+# Copyright (C) 2018 Michael Tremer                                           #
+#                                                                             #
+# This program is free software: you can redistribute it and/or modify        #
+# it under the terms of the GNU General Public License as published by        #
+# the Free Software Foundation, either version 3 of the License, or           #
+# (at your option) any later version.                                         #
+#                                                                             #
+# This program is distributed in the hope that it will be useful,             #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
+# GNU General Public License for more details.                                #
+#                                                                             #
+# You should have received a copy of the GNU General Public License           #
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
+#                                                                             #
+###############################################################################
+
+import sys
+
+import tools
+
+if not len(sys.argv) > 1:
+	sys.stderr.write("Not enough arguments\n")
+	sys.exit(2)
+
+path = sys.argv[1]
+
+database = tools.Database()
+for RIR in tools.RIRS:
+	rir = RIR()
+
+	database.import_rir(rir)
+
+database.write(path)
diff --git a/tools/__init__.py b/tools/__init__.py
index 5ab9a11..c4da84c 100644
--- a/tools/__init__.py
+++ b/tools/__init__.py
@@ -30,6 +30,8 @@ from .apnic import APNIC
 from .arin import ARIN
 from .ripe import RIPE
 
+from .database import Database
+
 RIRS = (
 	AFRINIC, APNIC, ARIN, RIPE,
 )
diff --git a/tools/database.py b/tools/database.py
new file mode 100644
index 0000000..d0a3669
--- /dev/null
+++ b/tools/database.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python3
+###############################################################################
+#                                                                             #
+# location-database - A database to determine someone's                       #
+#                     location on the Internet                                #
+# Copyright (C) 2018 Michael Tremer                                           #
+#                                                                             #
+# This program is free software: you can redistribute it and/or modify        #
+# it under the terms of the GNU General Public License as published by        #
+# the Free Software Foundation, either version 3 of the License, or           #
+# (at your option) any later version.                                         #
+#                                                                             #
+# This program is distributed in the hope that it will be useful,             #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
+# GNU General Public License for more details.                                #
+#                                                                             #
+# You should have received a copy of the GNU General Public License           #
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
+#                                                                             #
+###############################################################################
+
+import logging
+import re
+
+import location
+
+from . import util
+
+RE_AS = re.compile("^AS(\d+)$")
+
+class Database(object):
+	def __init__(self):
+		self.writer = location.Writer()
+
+		# XXX must set vendor, license and description
+
+	def write(self, path):
+		self.writer.write(path)
+
+	def import_rir(self, rir):
+		with open(rir.filename_asnums, "rb") as f:
+			for block in util.iterate_over_blocks(f):
+				self._parse_asnum_block(block)
+
+		with open(rir.filename_networks, "rb") as f:
+			for block in util.iterate_over_blocks(f):
+				self._parse_network_block(block)
+
+	def _parse_asnum_block(self, block):
+		asn  = None
+		name = None
+
+		for line in block:
+			key, val = util.split_line(line)
+
+			if key == "asnum":
+				m = RE_AS.match(val)
+				if m:
+					asn = int(m.group(1))
+
+			elif key == "name":
+				name = val
+
+		if asn and name:
+			a = self.writer.add_as(asn)
+			a.name = name
+
+			logging.debug("Added new AS: %s" % a)
+
+	def _parse_network_block(self, block):
+		network = None
+		asn     = None
+		country = None
+
+		for line in block:
+			key, val = util.split_line(line)
+
+			if key == "net":
+				network = val
+
+			elif key == "asnum":
+				m = RE_AS.match(val)
+				if m:
+					asn = int(m.group(1))
+
+			elif key == "country":
+				country = val
+
+		if network and country:
+			try:
+				n = self.writer.add_network(network)
+			except IndexError:
+				logging.warning("Skipping network %s, because one already exists at this address" % network)
+				return
+
+			# Save attributes
+			n.country_code = country
+			if asn:
+				n.asn = asn
+
+			logging.debug("Added new network: %s" % n)