From: Michael Tremer Date: Fri, 15 May 2020 09:23:26 +0000 (+0000) Subject: Drop all code from this repository X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5a352bd1822f81b623e1dcd44739b1c456eaa67f;p=location%2Flocation-database.git Drop all code from this repository This has been moved into libloc Signed-off-by: Michael Tremer --- diff --git a/compile-database b/compile-database deleted file mode 100755 index 85ff46c..0000000 --- a/compile-database +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2018 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -import sys - -import tools - -if not len(sys.argv) > 1: - sys.stderr.write("Not enough arguments\n") - sys.exit(2) - -path = sys.argv[1] - -database = tools.Database(vendor="IPFire Project") - -# Import countries -database.import_countries("countries.txt") - -# Import overlays first -for filename in ( - "country-overrides/override-a1.txt", - "country-overrides/override-a2.txt", - "country-overrides/override-a3.txt", - "country-overrides/override-other.txt", - ): - database.import_networks(filename) - -# Import all RIRs -for RIR in tools.RIRS: - rir = RIR() - - database.import_rir(rir) - -database.write(path) diff --git a/tools/__init__.py b/tools/__init__.py deleted file mode 100644 index 86b69f1..0000000 --- a/tools/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2018 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -import logging - -# Setup logging -logging.basicConfig(level=logging.INFO) - -from .afrinic import AFRINIC -from .apnic import APNIC -from .arin import ARIN -from .lacnic import LACNIC -from .ripe import RIPE - -from .database import Database - -RIRS = ( - AFRINIC, APNIC, ARIN, LACNIC, RIPE, -) diff --git a/tools/afrinic.py b/tools/afrinic.py deleted file mode 100644 index ad1059c..0000000 --- a/tools/afrinic.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2018 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -from . import base - -class AFRINIC(base.RIR): - name = "African Network Information Centre" - - database_urls = ( - "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", - ) - - whois_server = "whois.afrinic.net" diff --git a/tools/apnic.py b/tools/apnic.py deleted file mode 100644 index e6bb454..0000000 --- a/tools/apnic.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2018 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -from . import base - -class APNIC(base.RIR): - name = "Asia Pacific Network Information Centre" - - database_urls = ( - "http://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", - ) - - whois_server = "whois.apnic.net" diff --git a/tools/arin.py b/tools/arin.py deleted file mode 100644 index 729c83d..0000000 --- a/tools/arin.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2018 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -from . import base - -class ARIN(base.RIR): - name = "American Registry for Internet Numbers" - - database_urls = ( - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", - ) - - whois_server = "whois.arin.net" diff --git a/tools/base.py b/tools/base.py deleted file mode 100644 index 04f0016..0000000 --- a/tools/base.py +++ /dev/null @@ -1,412 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2018 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -import datetime -import ipaddress -import logging -import math -import os.path -import re -import sqlite3 -import struct -import subprocess - -from . import downloader -from . import util - -FILENAME_ASNUMS = "asnums.txt" -FILENMAE_NETWORKS = "networks.txt" - -FMT = "%-16s%s\n" - -RE_AS = re.compile(r"^(AS|as)(\d+)") - -INVALID_ADDRESSES = ( - "0.0.0.0", - "::/0", - "0::/0", -) - -class RIR(object): - name = None - database_urls = [] - - whois_server = None - - def __init__(self): - self.db = self._open_database(".cache.db") - - def __str__(self): - if self.name: - return "%s - %s" % (self.__class__.__name__, self.name) - - return self.__class__.__name__ - - def _open_database(self, path=None): - db = sqlite3.connect(path or ":memory:") - db.set_trace_callback(logging.debug) - - # Create tables - db.executescript(""" - CREATE TABLE IF NOT EXISTS whois_query_cache(query TEXT, response TEXT, - fetched_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP); - CREATE UNIQUE INDEX IF NOT EXISTS whois_query_cache_query - ON whois_query_cache(query); - """) - - return db - - @property - def parser(self): - return RIRParser - - def make_path(self, path): - return os.path.join(self.__class__.__name__, path) - - @property - def filename_asnums(self): - return self.make_path(FILENAME_ASNUMS) - - @property - def filename_networks(self): - return self.make_path(FILENMAE_NETWORKS) - - def update(self, directory): - p = self.parser(self) - - # Download all data and store it in memory - p.fetch_data() - - # Write the database to disk - p.export_database(directory) - - def _whois(self, query): - command = [ - "whois", query, - ] - - # Query a specific WHOIS server - if self.whois_server: - command += ["-h", self.whois_server] - - logging.info("Running command: %s" % " ".join(command)) - - try: - output = subprocess.check_output(command, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - logging.error("Could not run WHOIS query %s: %s" % (query, e.output)) - raise - - # Decode output - output = output.decode(errors="ignore") - - # Scan for any errors - for line in output.splitlines(): - if line.startswith("%ERROR:"): - raise OSError(output) - - return output - - def whois(self, query): - # Try fetching a response from the cache - with self.db as c: - res = c.execute("SELECT response, fetched_at FROM whois_query_cache \ - WHERE query = ?", (query,)) - - # Return any results - for row in res: - response, fetched_at = row - - logging.debug("Fetched response for %s from cache (%s)" - % (query, fetched_at)) - - return response - - # If we could not find anything, we will have to contact the whois server - response = self._whois(query) - - # Store the response in the database - with self.db as c: - c.execute("INSERT INTO whois_query_cache(query, response) \ - VALUES(?, ?)", (query, response)) - - # Commit changes to disk - self.db.commit() - - return response - - def get_name_for_asn(self, asn): - result = self.whois("AS%s" % asn) - - for line in result.splitlines(): - key, delim, value = line.partition(":") - if not value: - continue - - if key in ("org-name", "OrgName"): - return value.strip() - - -class RIRParser(object): - def __init__(self, rir): - self.rir = rir - - # Create a downloader to fetch data - self.downloader = downloader.Downloader() - - # Create a database to hold temporary data - self.db = self._make_database(":memory:") - - # Start time - self.start_time = datetime.datetime.utcnow() - - def _make_database(self, filename): - db = sqlite3.connect(filename) - db.set_trace_callback(logging.debug) - - # Create database layout - with db as cursor: - cursor.executescript(""" - CREATE TABLE IF NOT EXISTS autnums(asn INTEGER, country TEXT, org_id INTEGER, date DATE); - CREATE INDEX autnums_org_id ON autnums(org_id); - - CREATE TABLE IF NOT EXISTS inetnums(network TEXT, country TEXT, org_id INTEGER, - family INTEGER, address_start BLOB, address_end BLOB, prefix INTEGER, date DATE); - CREATE INDEX inetnums_sort ON inetnums(address_start); - """) - - return db - - def export_database(self, directory): - # Write all ASes - with open(self.rir.filename_asnums, "w") as f: - self._export_asnums(f) - - # Write all networks - with open(self.rir.filename_networks, "w") as f: - self._export_networks(f) - - def _write_header(self, f): - f.write("#\n") - f.write("# %s\n" % self.rir) - f.write("# Generated at %s\n" % self.start_time) - f.write("#\n\n") - - def fetch_data(self): - if not self.rir.database_urls: - raise NotImplementedError("Database URLs not set") - - # Parse entire database in one go - for url in self.rir.database_urls: - self.parse_url(url) - - self.db.commit() - - def parse_url(self, url): - with self.downloader.request(url) as r: - for line in r: - self.parse_line(line) - - def parse_line(self, line): - # Skip version line - if line.startswith("2"): - return - - # Skip comments - if line.startswith("#"): - return - - try: - registry, country_code, type, line = line.split("|", 3) - except: - logging.warning("Could not parse line: %s" % line) - return - - # Skip any lines that are for stats only - if country_code == "*": - return - - if type in ("ipv6", "ipv4"): - return self._parse_ip_line(country_code, type, line) - - elif type == "asn": - return self._parse_asn_line(country_code, line) - - else: - logging.warning("Unknown line type: %s" % type) - return - - def _parse_ip_line(self, country_code, type, line): - try: - address, prefix, date, status, org_id = line.split("|") - except ValueError: - org_id = None - - # Try parsing the line without org_id - try: - address, prefix, date, status = line.split("|") - except ValueError: - logging.warning("Unhandled line format: %s" % line) - return - - # Skip anything that isn't properly assigned - if not status in ("assigned", "allocated"): - return - - # Cast prefix into an integer - try: - prefix = int(prefix) - except: - logging.warning("Invalid prefix: %s" % prefix) - - # Fix prefix length for IPv4 - if type == "ipv4": - prefix = 32 - int(math.log(prefix, 2)) - - # Parse date - try: - date = datetime.datetime.strptime(date, "%Y%m%d") - except ValueError: - logging.warning("Could not parse date: %s" % date) - return - - # Try to parse the address - try: - network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False) - except ValueError: - logging.warning("Invalid IP address: %s" % address) - return - - with self.db as c: - # Get the first and last address of this network - address_start, address_end = int(network.network_address), int(network.broadcast_address) - - args = ( - "%s" % network, - country_code, - org_id, - network.version, - struct.pack(">QQ", address_start >> 64, address_start % (2 ** 64)), - struct.pack(">QQ", address_end >> 64, address_end % (2 ** 64)), - network.prefixlen, - date, - ) - - c.execute("INSERT INTO inetnums(network, country, org_id, \ - family, address_start, address_end, prefix, date) \ - VALUES(?, ?, ?, ?, ?, ?, ?, ?)", args) - - def _parse_asn_line(self, country_code, line): - try: - asn, dunno, date, status, org_id = line.split("|") - except ValueError: - org_id = None - - # Try parsing the line without org_id - try: - asn, dunno, date, status = line.split("|") - except ValueError: - logging.warning("Could not parse line: %s" % line) - return - - # Skip anything that isn't properly assigned - if not status in ("assigned", "allocated"): - return - - # Parse date - try: - date = datetime.datetime.strptime(date, "%Y%m%d") - except ValueError: - logging.warning("Could not parse date: %s" % date) - return - - with self.db as c: - args = ( - asn, - country_code, - org_id, - date, - ) - - c.execute("INSERT INTO autnums(asn, country, org_id, date) \ - VALUES(?, ?, ?, ?)", args) - - def _export_networks(self, f): - # Write header - self._write_header(f) - - with self.db as c: - # Write all networks - res = c.execute(""" - SELECT inetnums.network, - autnums.asn, - inetnums.address_start, - inetnums.country, - STRFTIME('%Y-%m-%d', inetnums.date) - FROM inetnums - LEFT JOIN autnums - WHERE inetnums.org_id = autnums.org_id - ORDER BY inetnums.address_start - """) - - for row in res: - net, asn, address_start, country, date = row - - f.write(FMT % ("net:", net)) - - if asn: - f.write(FMT % ("asnum:", "AS%s" % asn)) - - if country: - f.write(FMT % ("country:", country)) - - if date: - f.write(FMT % ("assigned:", date)) - - # End the block - f.write("\n") - - def _export_asnums(self, f): - # Write header - self._write_header(f) - - with self.db as c: - res = c.execute("SELECT DISTINCT autnums.asn, autnums.country, \ - STRFTIME('%Y-%m-%d', autnums.date) FROM autnums ORDER BY autnums.asn") - - for row in res: - asn, country, date = row - - f.write(FMT % ("asnum:", "AS%s" % asn)) - - name = self.rir.get_name_for_asn(asn) - if name: - f.write(FMT % ("name:", name)) - - if country: - f.write(FMT % ("country:", country)) - - if date: - f.write(FMT % ("assigned:", date)) - - # End block - f.write("\n") diff --git a/tools/database.py b/tools/database.py deleted file mode 100644 index 252229f..0000000 --- a/tools/database.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2018 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -import iso3166 -import logging -import re - -import location - -from . import util - -RE_AS = re.compile("^AS(\d+)$") - -class Database(object): - def __init__(self, vendor, description=None, license=None): - self.writer = location.Writer() - - # Set metadata - if vendor: - self.writer.vendor = vendor - - if description: - self.writer.description = description - - if license: - self.writer.license = license - - def write(self, path): - self.writer.write(path) - - def import_ases(self, filename): - with open(filename, "rb") as f: - for block in util.iterate_over_blocks(f): - self._parse_asnum_block(block) - - def import_networks(self, filename): - with open(filename, "rb") as f: - for block in util.iterate_over_blocks(f): - self._parse_network_block(block) - - def import_rir(self, rir): - self.import_ases(rir.filename_asnums) - self.import_networks(rir.filename_networks) - - def _parse_asnum_block(self, block): - asn = None - name = None - - for line in block: - key, val = util.split_line(line) - - if key == "asnum": - m = RE_AS.match(val) - if m: - asn = int(m.group(1)) - - elif key == "org": - name = val - - if asn and name: - a = self.writer.add_as(asn) - a.name = name - - logging.debug("Added new AS: %s" % a) - - def _parse_network_block(self, block): - network = None - asn = None - country = None - - for line in block: - key, val = util.split_line(line) - - if key == "net": - network = val - - elif key == "asnum": - m = RE_AS.match(val) - if m: - asn = int(m.group(1)) - - elif key == "country": - country = val - - if network and country: - try: - n = self.writer.add_network(network) - except IndexError: - logging.warning("Skipping network %s, because one already exists at this address" % network) - return - - # Translate some special country codes into flags - if country == "A1": - n.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY) - elif country == "A2": - n.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER) - elif country == "A3": - n.set_flag(location.NETWORK_FLAG_ANYCAST) - - # Set all other country codes - else: - n.country_code = country - - # Set ASN if available - if asn: - n.asn = asn - - logging.debug("Added new network: %s" % n) - - def import_countries(self, filename): - with open(filename) as f: - for line in f: - if line.startswith("#"): - continue - - try: - country_code, continent_code = line.split() - except: - logging.error("Could not parse line in %s: %s" (filename, line)) - raise - - c = self.writer.add_country(country_code) - - # Add continent code - if continent_code: - c.continent_code = continent_code - - # Add name - country = iso3166.countries_by_alpha2.get(country_code) - if country: - c.name = country.apolitical_name diff --git a/tools/downloader.py b/tools/downloader.py deleted file mode 100644 index 02f91a9..0000000 --- a/tools/downloader.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2018 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -import gzip -import logging -import urllib.request - -from . import util - -# Setup logger -log = logging.getLogger("downloader") -log.propagate = 1 - -class Downloader(object): - USER_AGENT = "location-database/1.0" - - def __init__(self): - self.proxy = None - - def set_proxy(self, url): - """ - Sets a HTTP proxy that is used to perform all requests - """ - log.info("Using proxy %s" % url) - self.proxy = url - - def request(self, url, data=None): - req = urllib.request.Request(url, data=data) - - # Configure proxy - if self.proxy: - req.set_proxy(self.proxy, "http") - - # Set User-Agent - if self.USER_AGENT: - req.add_header("User-Agent", self.USER_AGENT) - - return DownloaderContext(self, req) - - -class DownloaderContext(object): - def __init__(self, downloader, request): - self.downloader = downloader - self.request = request - - # Save the response object - self.response = None - - def __enter__(self): - log.info("Retrieving %s..." % self.request.full_url) - - # Send request - self.response = urllib.request.urlopen(self.request) - - # Log the response headers - log.debug("Response Headers:") - for header in self.headers: - log.debug(" %s: %s" % (header, self.get_header(header))) - - return self - - def __exit__(self, type, value, traceback): - pass - - def __iter__(self): - """ - Makes the object iterable by going through each block - """ - # Store body - body = self.body - - while True: - line = body.readline() - if not line: - break - - # Decode the line - line = line.decode() - - # Strip the ending - yield line.rstrip() - - @property - def headers(self): - if self.response: - return self.response.headers - - def get_header(self, name): - if self.headers: - return self.headers.get(name) - - @property - def body(self): - """ - Returns a file-like object with the decoded content - of the response. - """ - # Return the response by default - return self.response - - -if __name__ == "__main__": - import sys - - # Enable debug logging - logging.basicConfig(level=logging.DEBUG) - - d = Downloader() - - for url in sys.argv[1:]: - print("Downloading %s..." % url) - - with d.request(url) as r: - for line in r: - print(line) diff --git a/tools/lacnic.py b/tools/lacnic.py deleted file mode 100644 index 5682873..0000000 --- a/tools/lacnic.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2019 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -import ipaddress -import logging -import math -import sqlite3 -import struct - -from . import base - -class LACNIC(base.RIR): - name = "Latin America and Caribbean Network Information Centre" - - database_urls = ( - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - ) - - whois_server = "whois.lacnic.net" diff --git a/tools/ripe.py b/tools/ripe.py deleted file mode 100644 index 4f10a4a..0000000 --- a/tools/ripe.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2018 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -from . import base - -class RIPE(base.RIR): - name = "Réseaux IP Européens" - - database_urls = ( - "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", - ) - - whois_server = "whois.ripe.net" diff --git a/tools/util.py b/tools/util.py deleted file mode 100644 index 260b924..0000000 --- a/tools/util.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2018 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -def iterate_over_blocks(f, charsets=("utf-8", "latin1")): - block = [] - - for line in f: - # Convert to string - for charset in charsets: - try: - line = line.decode(charset) - except UnicodeDecodeError: - continue - else: - break - - # Strip line-endings - line = line.rstrip() - - # Skip commented lines - if line.startswith("#") or line.startswith("%"): - continue - - # Remove any comments at the end of line - line, hash, comment = line.partition("#") - - if comment: - # Strip any whitespace before the comment - line = line.rstrip() - - # If the line is now empty, we move on - if not line: - continue - - if line: - block.append(line) - continue - - # End the block on an empty line - if block: - yield block - - # Reset the block - block = [] - -def split_line(line): - key, colon, val = line.partition(":") - - # Strip any excess space - key = key.strip() - val = val.strip() - - return key, val diff --git a/update-rirs b/update-rirs deleted file mode 100755 index bea2b01..0000000 --- a/update-rirs +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# location-database - A database to determine someone's # -# location on the Internet # -# Copyright (C) 2018 Michael Tremer # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -import sys - -import tools - -# The user may pass a list of RIRs on the command line that should be updated -update_only = sys.argv[1:] - -for RIR in tools.RIRS: - name = RIR.__name__ - - # Skip RIRs that should not be updated - if update_only and not name in update_only: - continue - - rir = RIR() - - # Update all records from the database - rir.update(name)