From: Michael Tremer Date: Mon, 14 Jan 2019 09:33:03 +0000 (+0000) Subject: Add parser for LACNIC X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e861ab14216a26e6afcbd4f1d582ec146c9c44b4;p=location%2Flocation-database.git Add parser for LACNIC Signed-off-by: Michael Tremer --- diff --git a/tools/__init__.py b/tools/__init__.py index c4da84c..86b69f1 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -28,10 +28,11 @@ logging.basicConfig(level=logging.INFO) from .afrinic import AFRINIC from .apnic import APNIC from .arin import ARIN +from .lacnic import LACNIC from .ripe import RIPE from .database import Database RIRS = ( - AFRINIC, APNIC, ARIN, RIPE, + AFRINIC, APNIC, ARIN, LACNIC, RIPE, ) diff --git a/tools/downloader.py b/tools/downloader.py index bd24ba0..56714ef 100644 --- a/tools/downloader.py +++ b/tools/downloader.py @@ -43,7 +43,7 @@ class Downloader(object): log.info("Using proxy %s" % url) self.proxy = url - def request(self, url, data=None): + def request(self, url, data=None, return_blocks=True): req = urllib.request.Request(url, data=data) # Configure proxy @@ -54,14 +54,17 @@ class Downloader(object): if self.USER_AGENT: req.add_header("User-Agent", self.USER_AGENT) - return DownloaderContext(self, req) + return DownloaderContext(self, req, return_blocks=return_blocks) class DownloaderContext(object): - def __init__(self, downloader, request): + def __init__(self, downloader, request, return_blocks=True): self.downloader = downloader self.request = request + # Should we return one block or a single line? + self.return_blocks = return_blocks + # Save the response object self.response = None @@ -85,7 +88,22 @@ class DownloaderContext(object): """ Makes the object iterable by going through each block """ - return util.iterate_over_blocks(self.body) + if self.return_blocks: + return util.iterate_over_blocks(self.body) + + # Store body + body = self.body + + while True: + line = body.readline() + if not line: + break + + # Decode the line + line = line.decode() + + # Strip the ending + yield line.rstrip() @property def headers(self): diff --git a/tools/lacnic.py b/tools/lacnic.py new file mode 100644 index 0000000..13f0b8d --- /dev/null +++ b/tools/lacnic.py @@ -0,0 +1,194 @@ +#!/usr/bin/python3 +############################################################################### +# # +# location-database - A database to determine someone's # +# location on the Internet # +# Copyright (C) 2019 Michael Tremer # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import ipaddress +import logging +import math +import sqlite3 +import struct + +from . import base + +class LACNIC(base.RIR): + name = "Latin America and Caribbean Network Information Centre" + + database_urls = ( + "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", + ) + + @property + def parser(self): + return LACNICParser + + +class LACNICParser(base.RIRParser): + def _make_database(self, filename): + db = sqlite3.connect(filename) + + # Create database layout + with db as cursor: + cursor.executescript(""" + CREATE TABLE IF NOT EXISTS autnums(asn INTEGER, country TEXT, org_id INTEGER); + CREATE INDEX autnums_org_id ON autnums(org_id); + + CREATE TABLE IF NOT EXISTS inetnums(network TEXT, country TEXT, org_id INTEGER, + family INTEGER, address_start BLOB, address_end BLOB, prefix INTEGER); + CREATE INDEX inetnums_sort ON inetnums(address_start); + """) + + return db + + def parse_url(self, url): + with self.downloader.request(url, return_blocks=False) as r: + for line in r: + self.parse_line(line) + + def parse_line(self, line): + try: + lacnic, country_code, type, line = line.split("|", 3) + except: + logging.warning("Could not parse line: %s" % line) + return + + # Skip any lines with addresses that are not allocated + if not "|allocated|" in line: + return + + if type in ("ipv6", "ipv4"): + return self._parse_ip_line(country_code, type, line) + + elif type == "asn": + return self._parse_asn_line(country_code, line) + + else: + logging.warning("Unknown line type: %s" % type) + return + + def _parse_ip_line(self, country_code, type, line): + try: + address, prefix, date, status, org_id = line.split("|") + except: + logging.warning("Unhandled line format: %s" % line) + return + + # Cast prefix into an integer + try: + prefix = int(prefix) + except: + logging.warning("Invalid prefix: %s" % prefix) + + # Fix prefix length for IPv4 + if type == "ipv4": + prefix = 32 - int(math.log(prefix, 2)) + + # Try to parse the address + try: + network = ipaddress.ip_network("%s/%s" % (address, prefix)) + except ValueError: + raise + logging.warning("Invalid IP address: %s" % address) + return + + with self.db as c: + # Get the first and last address of this network + address_start, address_end = int(network.network_address), int(network.broadcast_address) + + args = ( + "%s" % network, + country_code, + org_id, + network.version, + struct.pack(">QQ", address_start >> 64, address_start % (2 ** 64)), + struct.pack(">QQ", address_end >> 64, address_end % (2 ** 64)), + network.prefixlen, + ) + + c.execute("INSERT INTO inetnums(network, country, org_id, \ + family, address_start, address_end, prefix) VALUES(?, ?, ?, ?, ?, ?, ?)", args) + + def _parse_asn_line(self, country_code, line): + try: + asn, dunno, date, status, org_id = line.split("|") + except: + logging.warning("Could not parse line: %s" % line) + return + + with self.db as c: + args = ( + asn, + country_code, + org_id, + ) + + c.execute("INSERT INTO autnums(asn, country, org_id) VALUES(?, ?, ?)", args) + + def _export_networks(self, f): + # Write header + self._write_header(f) + + with self.db as c: + # Write all networks + res = c.execute(""" + SELECT inetnums.network, + autnums.asn, + inetnums.address_start, + inetnums.country + FROM inetnums + LEFT JOIN autnums + WHERE inetnums.org_id = autnums.org_id + ORDER BY inetnums.address_start + """) + + for row in res: + net, asn, address_start, country = row + + f.write(base.FMT % ("net:", net)) + + if asn: + f.write(base.FMT % ("asnum:", "AS%s" % asn)) + + if country: + f.write(base.FMT % ("country:", country)) + + # End the block + f.write("\n") + + def _export_asnums(self, f): + # Write header + self._write_header(f) + + with self.db as c: + res = c.execute("SELECT DISTINCT autnums.asn, autnums.country \ + FROM autnums ORDER BY autnums.asn") + + for row in res: + asn, country = row + + f.write(base.FMT % ("asnum:", "AS%s" % asn)) + + if country: + f.write(base.FMT % ("country:", country)) + + # End block + f.write("\n") + +