log.info("Using proxy %s" % url)
self.proxy = url
- def request(self, url, data=None):
+ def request(self, url, data=None, return_blocks=True):
req = urllib.request.Request(url, data=data)
# Configure proxy
if self.USER_AGENT:
req.add_header("User-Agent", self.USER_AGENT)
- return DownloaderContext(self, req)
+ return DownloaderContext(self, req, return_blocks=return_blocks)
class DownloaderContext(object):
- def __init__(self, downloader, request):
+ def __init__(self, downloader, request, return_blocks=True):
self.downloader = downloader
self.request = request
+ # Should we return one block or a single line?
+ self.return_blocks = return_blocks
+
# Save the response object
self.response = None
"""
Makes the object iterable by going through each block
"""
- return util.iterate_over_blocks(self.body)
+ if self.return_blocks:
+ return util.iterate_over_blocks(self.body)
+
+ # Store body
+ body = self.body
+
+ while True:
+ line = body.readline()
+ if not line:
+ break
+
+ # Decode the line
+ line = line.decode()
+
+ # Strip the ending
+ yield line.rstrip()
@property
def headers(self):
--- /dev/null
+#!/usr/bin/python3
+###############################################################################
+# #
+# location-database - A database to determine someone's #
+# location on the Internet #
+# Copyright (C) 2019 Michael Tremer #
+# #
+# This program is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# This program is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with this program. If not, see <http://www.gnu.org/licenses/>. #
+# #
+###############################################################################
+
+import ipaddress
+import logging
+import math
+import sqlite3
+import struct
+
+from . import base
+
+class LACNIC(base.RIR):
+ name = "Latin America and Caribbean Network Information Centre"
+
+ database_urls = (
+ "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
+ )
+
+ @property
+ def parser(self):
+ return LACNICParser
+
+
+class LACNICParser(base.RIRParser):
+ def _make_database(self, filename):
+ db = sqlite3.connect(filename)
+
+ # Create database layout
+ with db as cursor:
+ cursor.executescript("""
+ CREATE TABLE IF NOT EXISTS autnums(asn INTEGER, country TEXT, org_id INTEGER);
+ CREATE INDEX autnums_org_id ON autnums(org_id);
+
+ CREATE TABLE IF NOT EXISTS inetnums(network TEXT, country TEXT, org_id INTEGER,
+ family INTEGER, address_start BLOB, address_end BLOB, prefix INTEGER);
+ CREATE INDEX inetnums_sort ON inetnums(address_start);
+ """)
+
+ return db
+
+ def parse_url(self, url):
+ with self.downloader.request(url, return_blocks=False) as r:
+ for line in r:
+ self.parse_line(line)
+
+ def parse_line(self, line):
+ try:
+ lacnic, country_code, type, line = line.split("|", 3)
+ except:
+ logging.warning("Could not parse line: %s" % line)
+ return
+
+ # Skip any lines with addresses that are not allocated
+ if not "|allocated|" in line:
+ return
+
+ if type in ("ipv6", "ipv4"):
+ return self._parse_ip_line(country_code, type, line)
+
+ elif type == "asn":
+ return self._parse_asn_line(country_code, line)
+
+ else:
+ logging.warning("Unknown line type: %s" % type)
+ return
+
+ def _parse_ip_line(self, country_code, type, line):
+ try:
+ address, prefix, date, status, org_id = line.split("|")
+ except:
+ logging.warning("Unhandled line format: %s" % line)
+ return
+
+ # Cast prefix into an integer
+ try:
+ prefix = int(prefix)
+ except:
+ logging.warning("Invalid prefix: %s" % prefix)
+
+ # Fix prefix length for IPv4
+ if type == "ipv4":
+ prefix = 32 - int(math.log(prefix, 2))
+
+ # Try to parse the address
+ try:
+ network = ipaddress.ip_network("%s/%s" % (address, prefix))
+ except ValueError:
+ raise
+ logging.warning("Invalid IP address: %s" % address)
+ return
+
+ with self.db as c:
+ # Get the first and last address of this network
+ address_start, address_end = int(network.network_address), int(network.broadcast_address)
+
+ args = (
+ "%s" % network,
+ country_code,
+ org_id,
+ network.version,
+ struct.pack(">QQ", address_start >> 64, address_start % (2 ** 64)),
+ struct.pack(">QQ", address_end >> 64, address_end % (2 ** 64)),
+ network.prefixlen,
+ )
+
+ c.execute("INSERT INTO inetnums(network, country, org_id, \
+ family, address_start, address_end, prefix) VALUES(?, ?, ?, ?, ?, ?, ?)", args)
+
+ def _parse_asn_line(self, country_code, line):
+ try:
+ asn, dunno, date, status, org_id = line.split("|")
+ except:
+ logging.warning("Could not parse line: %s" % line)
+ return
+
+ with self.db as c:
+ args = (
+ asn,
+ country_code,
+ org_id,
+ )
+
+ c.execute("INSERT INTO autnums(asn, country, org_id) VALUES(?, ?, ?)", args)
+
+ def _export_networks(self, f):
+ # Write header
+ self._write_header(f)
+
+ with self.db as c:
+ # Write all networks
+ res = c.execute("""
+ SELECT inetnums.network,
+ autnums.asn,
+ inetnums.address_start,
+ inetnums.country
+ FROM inetnums
+ LEFT JOIN autnums
+ WHERE inetnums.org_id = autnums.org_id
+ ORDER BY inetnums.address_start
+ """)
+
+ for row in res:
+ net, asn, address_start, country = row
+
+ f.write(base.FMT % ("net:", net))
+
+ if asn:
+ f.write(base.FMT % ("asnum:", "AS%s" % asn))
+
+ if country:
+ f.write(base.FMT % ("country:", country))
+
+ # End the block
+ f.write("\n")
+
+ def _export_asnums(self, f):
+ # Write header
+ self._write_header(f)
+
+ with self.db as c:
+ res = c.execute("SELECT DISTINCT autnums.asn, autnums.country \
+ FROM autnums ORDER BY autnums.asn")
+
+ for row in res:
+ asn, country = row
+
+ f.write(base.FMT % ("asnum:", "AS%s" % asn))
+
+ if country:
+ f.write(base.FMT % ("country:", country))
+
+ # End block
+ f.write("\n")
+
+
projects / location / location-database.git / commitdiff
