Add script to export data into a location database

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/compile-database b/compile-database
new file mode 100755 (executable)
index 0000000..f225186
--- /dev/null
+++ b/compile-database
@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+###############################################################################
+#                                                                             #
+# location-database - A database to determine someone's                       #
+#                     location on the Internet                                #
+# Copyright (C) 2018 Michael Tremer                                           #
+#                                                                             #
+# This program is free software: you can redistribute it and/or modify        #
+# it under the terms of the GNU General Public License as published by        #
+# the Free Software Foundation, either version 3 of the License, or           #
+# (at your option) any later version.                                         #
+#                                                                             #
+# This program is distributed in the hope that it will be useful,             #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
+# GNU General Public License for more details.                                #
+#                                                                             #
+# You should have received a copy of the GNU General Public License           #
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
+#                                                                             #
+###############################################################################
+
+import sys
+
+import tools
+
+if not len(sys.argv) > 1:
+       sys.stderr.write("Not enough arguments\n")
+       sys.exit(2)
+
+path = sys.argv[1]
+
+database = tools.Database()
+for RIR in tools.RIRS:
+       rir = RIR()
+
+       database.import_rir(rir)
+
+database.write(path)

diff --git a/tools/__init__.py b/tools/__init__.py
index 5ab9a112158a8da30136d74be0dc1c012c5a24af..c4da84c935bc84a7247bbe3391f708c60e145238 100644 (file)
--- a/tools/__init__.py
+++ b/tools/__init__.py
@@ -30,6 +30,8 @@ from .apnic import APNIC
 from .arin import ARIN
 from .ripe import RIPE
 
+from .database import Database
+
 RIRS = (
        AFRINIC, APNIC, ARIN, RIPE,
 )

diff --git a/tools/database.py b/tools/database.py
new file mode 100644 (file)
index 0000000..d0a3669
--- /dev/null
+++ b/tools/database.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python3
+###############################################################################
+#                                                                             #
+# location-database - A database to determine someone's                       #
+#                     location on the Internet                                #
+# Copyright (C) 2018 Michael Tremer                                           #
+#                                                                             #
+# This program is free software: you can redistribute it and/or modify        #
+# it under the terms of the GNU General Public License as published by        #
+# the Free Software Foundation, either version 3 of the License, or           #
+# (at your option) any later version.                                         #
+#                                                                             #
+# This program is distributed in the hope that it will be useful,             #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
+# GNU General Public License for more details.                                #
+#                                                                             #
+# You should have received a copy of the GNU General Public License           #
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
+#                                                                             #
+###############################################################################
+
+import logging
+import re
+
+import location
+
+from . import util
+
+RE_AS = re.compile("^AS(\d+)$")
+
+class Database(object):
+       def __init__(self):
+               self.writer = location.Writer()
+
+               # XXX must set vendor, license and description
+
+       def write(self, path):
+               self.writer.write(path)
+
+       def import_rir(self, rir):
+               with open(rir.filename_asnums, "rb") as f:
+                       for block in util.iterate_over_blocks(f):
+                               self._parse_asnum_block(block)
+
+               with open(rir.filename_networks, "rb") as f:
+                       for block in util.iterate_over_blocks(f):
+                               self._parse_network_block(block)
+
+       def _parse_asnum_block(self, block):
+               asn  = None
+               name = None
+
+               for line in block:
+                       key, val = util.split_line(line)
+
+                       if key == "asnum":
+                               m = RE_AS.match(val)
+                               if m:
+                                       asn = int(m.group(1))
+
+                       elif key == "name":
+                               name = val
+
+               if asn and name:
+                       a = self.writer.add_as(asn)
+                       a.name = name
+
+                       logging.debug("Added new AS: %s" % a)
+
+       def _parse_network_block(self, block):
+               network = None
+               asn     = None
+               country = None
+
+               for line in block:
+                       key, val = util.split_line(line)
+
+                       if key == "net":
+                               network = val
+
+                       elif key == "asnum":
+                               m = RE_AS.match(val)
+                               if m:
+                                       asn = int(m.group(1))
+
+                       elif key == "country":
+                               country = val
+
+               if network and country:
+                       try:
+                               n = self.writer.add_network(network)
+                       except IndexError:
+                               logging.warning("Skipping network %s, because one already exists at this address" % network)
+                               return
+
+                       # Save attributes
+                       n.country_code = country
+                       if asn:
+                               n.asn = asn
+
+                       logging.debug("Added new network: %s" % n)