+++ /dev/null
-From: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu, 14 Apr 2022 18:31:56 +0000
-Subject: Make sources around that we can run tests without location installed
-
-In order to run the test suite, we need to make the Python module
-loadable from the build directory so that we first of all test the right
-code and that it just works without running "make install" first.
-
-Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
----
- .gitignore | 5 +-
- Makefile.am | 37 +-
- po/POTFILES.in | 18 +-
- src/python/__init__.py.in | 26 -
- src/python/database.py | 213 -----
- src/python/downloader.py | 211 -----
- src/python/export.py | 291 -------
- src/python/i18n.py | 26 -
- src/python/importer.py | 250 ------
- src/python/location-importer.in | 1535 -------------------------------------
- src/python/location.in | 644 ----------------
- src/python/location/__init__.py | 24 +
- src/python/location/database.py | 213 +++++
- src/python/location/downloader.py | 211 +++++
- src/python/location/export.py | 291 +++++++
- src/python/location/i18n.py | 26 +
- src/python/location/importer.py | 250 ++++++
- src/python/location/logger.py | 46 ++
- src/python/logger.py | 46 --
- src/scripts/location-importer.in | 1535 +++++++++++++++++++++++++++++++++++++
- src/scripts/location.in | 644 ++++++++++++++++
- 21 files changed, 3266 insertions(+), 3276 deletions(-)
- delete mode 100644 src/python/__init__.py.in
- delete mode 100644 src/python/database.py
- delete mode 100644 src/python/downloader.py
- delete mode 100644 src/python/export.py
- delete mode 100644 src/python/i18n.py
- delete mode 100644 src/python/importer.py
- delete mode 100644 src/python/location-importer.in
- delete mode 100644 src/python/location.in
- create mode 100644 src/python/location/__init__.py
- create mode 100644 src/python/location/database.py
- create mode 100644 src/python/location/downloader.py
- create mode 100644 src/python/location/export.py
- create mode 100644 src/python/location/i18n.py
- create mode 100644 src/python/location/importer.py
- create mode 100644 src/python/location/logger.py
- delete mode 100644 src/python/logger.py
- create mode 100644 src/scripts/location-importer.in
- create mode 100644 src/scripts/location.in
-
-diff --git a/.gitignore b/.gitignore
-index f04b70e..20bc895 100644
---- a/.gitignore
-+++ b/.gitignore
-@@ -15,9 +15,8 @@ Makefile.in
- /*.db.xz
- /libtool
- /stamp-h1
--/src/python/location
--/src/python/location-importer
--/src/python/__init__.py
-+/src/scripts/location
-+/src/scripts/location-importer
- /src/systemd/location-update.service
- /src/systemd/location-update.timer
- /test.db
-diff --git a/Makefile.am b/Makefile.am
-index 983cb4a..38ce961 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -175,21 +175,13 @@ CLEANFILES += \
- src/libloc.pc
-
- dist_pkgpython_PYTHON = \
-- src/python/database.py \
-- src/python/downloader.py \
-- src/python/export.py \
-- src/python/i18n.py \
-- src/python/importer.py \
-- src/python/logger.py
--
--pkgpython_PYTHON = \
-- src/python/__init__.py
--
--EXTRA_DIST += \
-- src/python/__init__.py.in
--
--CLEANFILES += \
-- src/python/__init__.py
-+ src/python/location/__init__.py \
-+ src/python/location/database.py \
-+ src/python/location/downloader.py \
-+ src/python/location/export.py \
-+ src/python/location/i18n.py \
-+ src/python/location/importer.py \
-+ src/python/location/logger.py
-
- pyexec_LTLIBRARIES = \
- src/python/_location.la
-@@ -275,16 +267,16 @@ uninstall-perl:
- $(DESTDIR)/$(prefix)/man/man3/Location.3pm
-
- bin_SCRIPTS = \
-- src/python/location \
-- src/python/location-importer
-+ src/scripts/location \
-+ src/scripts/location-importer
-
- EXTRA_DIST += \
-- src/python/location.in \
-- src/python/location-importer.in
-+ src/scripts/location.in \
-+ src/scripts/location-importer.in
-
- CLEANFILES += \
-- src/python/location \
-- src/python/location-importer
-+ src/scripts/location \
-+ src/scripts/location-importer
-
- # ------------------------------------------------------------------------------
-
-@@ -321,6 +313,7 @@ TESTS_LDADD = \
- src/libloc-internal.la
-
- TESTS_ENVIRONMENT = \
-+ PYTHONPATH=$(abs_srcdir)/src/python:$(abs_builddir)/src/python/.libs \
- TEST_DATA_DIR="$(abs_top_srcdir)/tests/data"
-
- TESTS = \
-@@ -334,7 +327,7 @@ CLEANFILES += \
- testdata.db
-
- testdata.db: examples/python/create-database.py
-- PYTHONPATH=$(abs_builddir)/src/python/.libs \
-+ PYTHONPATH=$(abs_srcdir)/src/python:$(abs_builddir)/src/python/.libs \
- ABS_SRCDIR="$(abs_srcdir)" \
- $(PYTHON) $< $@
-
-diff --git a/po/POTFILES.in b/po/POTFILES.in
-index 5d2cc46..5f5afa8 100644
---- a/po/POTFILES.in
-+++ b/po/POTFILES.in
-@@ -1,12 +1,12 @@
- src/libloc.pc.in
--src/python/__init__.py.in
--src/python/database.py
--src/python/downloader.py
--src/python/export.py
--src/python/i18n.py
--src/python/importer.py
--src/python/location-importer.in
--src/python/location.in
--src/python/logger.py
-+src/python/location/__init__.py
-+src/python/location/database.py
-+src/python/location/downloader.py
-+src/python/location/export.py
-+src/python/location/i18n.py
-+src/python/location/importer.py
-+src/python/location/logger.py
-+src/scripts/location-importer.in
-+src/scripts/location.in
- src/systemd/location-update.service.in
- src/systemd/location-update.timer.in
-diff --git a/src/python/__init__.py.in b/src/python/__init__.py.in
-deleted file mode 100644
-index bd94d35..0000000
---- a/src/python/__init__.py.in
-+++ /dev/null
-@@ -1,26 +0,0 @@
--#!/usr/bin/python3
--###############################################################################
--# #
--# libloc - A library to determine the location of someone on the Internet #
--# #
--# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
--# #
--# This library is free software; you can redistribute it and/or #
--# modify it under the terms of the GNU Lesser General Public #
--# License as published by the Free Software Foundation; either #
--# version 2.1 of the License, or (at your option) any later version. #
--# #
--# This library is distributed in the hope that it will be useful, #
--# but WITHOUT ANY WARRANTY; without even the implied warranty of #
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
--# Lesser General Public License for more details. #
--# #
--###############################################################################
--
--__version__ = "@VERSION@"
--
--# Import everything from the C module
--from _location import *
--
--# Initialise logging
--from . import logger
-diff --git a/src/python/database.py b/src/python/database.py
-deleted file mode 100644
-index 5d79941..0000000
---- a/src/python/database.py
-+++ /dev/null
-@@ -1,213 +0,0 @@
--#!/usr/bin/env python
--
--"""
-- A lightweight wrapper around psycopg2.
--
-- Originally part of the Tornado framework. The tornado.database module
-- is slated for removal in Tornado 3.0, and it is now available separately
-- as torndb.
--"""
--
--import logging
--import psycopg2
--
--log = logging.getLogger("location.database")
--log.propagate = 1
--
--class Connection(object):
-- """
-- A lightweight wrapper around MySQLdb DB-API connections.
--
-- The main value we provide is wrapping rows in a dict/object so that
-- columns can be accessed by name. Typical usage::
--
-- db = torndb.Connection("localhost", "mydatabase")
-- for article in db.query("SELECT * FROM articles"):
-- print article.title
--
-- Cursors are hidden by the implementation, but other than that, the methods
-- are very similar to the DB-API.
--
-- We explicitly set the timezone to UTC and the character encoding to
-- UTF-8 on all connections to avoid time zone and encoding errors.
-- """
-- def __init__(self, host, database, user=None, password=None):
-- self.host = host
-- self.database = database
--
-- self._db = None
-- self._db_args = {
-- "host" : host,
-- "database" : database,
-- "user" : user,
-- "password" : password,
-- "sslmode" : "require",
-- }
--
-- try:
-- self.reconnect()
-- except Exception:
-- log.error("Cannot connect to database on %s", self.host, exc_info=True)
--
-- def __del__(self):
-- self.close()
--
-- def close(self):
-- """
-- Closes this database connection.
-- """
-- if getattr(self, "_db", None) is not None:
-- self._db.close()
-- self._db = None
--
-- def reconnect(self):
-- """
-- Closes the existing database connection and re-opens it.
-- """
-- self.close()
--
-- self._db = psycopg2.connect(**self._db_args)
-- self._db.autocommit = True
--
-- # Initialize the timezone setting.
-- self.execute("SET TIMEZONE TO 'UTC'")
--
-- def query(self, query, *parameters, **kwparameters):
-- """
-- Returns a row list for the given query and parameters.
-- """
-- cursor = self._cursor()
-- try:
-- self._execute(cursor, query, parameters, kwparameters)
-- column_names = [d[0] for d in cursor.description]
-- return [Row(zip(column_names, row)) for row in cursor]
-- finally:
-- cursor.close()
--
-- def get(self, query, *parameters, **kwparameters):
-- """
-- Returns the first row returned for the given query.
-- """
-- rows = self.query(query, *parameters, **kwparameters)
-- if not rows:
-- return None
-- elif len(rows) > 1:
-- raise Exception("Multiple rows returned for Database.get() query")
-- else:
-- return rows[0]
--
-- def execute(self, query, *parameters, **kwparameters):
-- """
-- Executes the given query, returning the lastrowid from the query.
-- """
-- return self.execute_lastrowid(query, *parameters, **kwparameters)
--
-- def execute_lastrowid(self, query, *parameters, **kwparameters):
-- """
-- Executes the given query, returning the lastrowid from the query.
-- """
-- cursor = self._cursor()
-- try:
-- self._execute(cursor, query, parameters, kwparameters)
-- return cursor.lastrowid
-- finally:
-- cursor.close()
--
-- def execute_rowcount(self, query, *parameters, **kwparameters):
-- """
-- Executes the given query, returning the rowcount from the query.
-- """
-- cursor = self._cursor()
-- try:
-- self._execute(cursor, query, parameters, kwparameters)
-- return cursor.rowcount
-- finally:
-- cursor.close()
--
-- def executemany(self, query, parameters):
-- """
-- Executes the given query against all the given param sequences.
--
-- We return the lastrowid from the query.
-- """
-- return self.executemany_lastrowid(query, parameters)
--
-- def executemany_lastrowid(self, query, parameters):
-- """
-- Executes the given query against all the given param sequences.
--
-- We return the lastrowid from the query.
-- """
-- cursor = self._cursor()
-- try:
-- cursor.executemany(query, parameters)
-- return cursor.lastrowid
-- finally:
-- cursor.close()
--
-- def executemany_rowcount(self, query, parameters):
-- """
-- Executes the given query against all the given param sequences.
--
-- We return the rowcount from the query.
-- """
-- cursor = self._cursor()
--
-- try:
-- cursor.executemany(query, parameters)
-- return cursor.rowcount
-- finally:
-- cursor.close()
--
-- def _ensure_connected(self):
-- if self._db is None:
-- log.warning("Database connection was lost...")
--
-- self.reconnect()
--
-- def _cursor(self):
-- self._ensure_connected()
-- return self._db.cursor()
--
-- def _execute(self, cursor, query, parameters, kwparameters):
-- log.debug("SQL Query: %s" % (query % (kwparameters or parameters)))
--
-- try:
-- return cursor.execute(query, kwparameters or parameters)
-- except (OperationalError, psycopg2.ProgrammingError):
-- log.error("Error connecting to database on %s", self.host)
-- self.close()
-- raise
--
-- def transaction(self):
-- return Transaction(self)
--
--
--class Row(dict):
-- """A dict that allows for object-like property access syntax."""
-- def __getattr__(self, name):
-- try:
-- return self[name]
-- except KeyError:
-- raise AttributeError(name)
--
--
--class Transaction(object):
-- def __init__(self, db):
-- self.db = db
--
-- self.db.execute("START TRANSACTION")
--
-- def __enter__(self):
-- return self
--
-- def __exit__(self, exctype, excvalue, traceback):
-- if exctype is not None:
-- self.db.execute("ROLLBACK")
-- else:
-- self.db.execute("COMMIT")
--
--
--# Alias some common exceptions
--IntegrityError = psycopg2.IntegrityError
--OperationalError = psycopg2.OperationalError
-diff --git a/src/python/downloader.py b/src/python/downloader.py
-deleted file mode 100644
-index 05f7872..0000000
---- a/src/python/downloader.py
-+++ /dev/null
-@@ -1,211 +0,0 @@
--#!/usr/bin/python3
--###############################################################################
--# #
--# libloc - A library to determine the location of someone on the Internet #
--# #
--# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
--# #
--# This library is free software; you can redistribute it and/or #
--# modify it under the terms of the GNU Lesser General Public #
--# License as published by the Free Software Foundation; either #
--# version 2.1 of the License, or (at your option) any later version. #
--# #
--# This library is distributed in the hope that it will be useful, #
--# but WITHOUT ANY WARRANTY; without even the implied warranty of #
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
--# Lesser General Public License for more details. #
--# #
--###############################################################################
--
--import logging
--import lzma
--import os
--import random
--import stat
--import tempfile
--import time
--import urllib.error
--import urllib.parse
--import urllib.request
--
--from . import __version__
--from _location import Database, DATABASE_VERSION_LATEST
--
--DATABASE_FILENAME = "location.db.xz"
--MIRRORS = (
-- "https://location.ipfire.org/databases/",
--)
--
--# Initialise logging
--log = logging.getLogger("location.downloader")
--log.propagate = 1
--
--class Downloader(object):
-- def __init__(self, version=DATABASE_VERSION_LATEST, mirrors=None):
-- self.version = version
--
-- # Set mirrors or use defaults
-- self.mirrors = list(mirrors or MIRRORS)
--
-- # Randomize mirrors
-- random.shuffle(self.mirrors)
--
-- # Get proxies from environment
-- self.proxies = self._get_proxies()
--
-- def _get_proxies(self):
-- proxies = {}
--
-- for protocol in ("https", "http"):
-- proxy = os.environ.get("%s_proxy" % protocol, None)
--
-- if proxy:
-- proxies[protocol] = proxy
--
-- return proxies
--
-- def _make_request(self, url, baseurl=None, headers={}):
-- if baseurl:
-- url = urllib.parse.urljoin(baseurl, url)
--
-- req = urllib.request.Request(url, method="GET")
--
-- # Update headers
-- headers.update({
-- "User-Agent" : "location/%s" % __version__,
-- })
--
-- # Set headers
-- for header in headers:
-- req.add_header(header, headers[header])
--
-- # Set proxies
-- for protocol in self.proxies:
-- req.set_proxy(self.proxies[protocol], protocol)
--
-- return req
--
-- def _send_request(self, req, **kwargs):
-- # Log request headers
-- log.debug("HTTP %s Request to %s" % (req.method, req.host))
-- log.debug(" URL: %s" % req.full_url)
-- log.debug(" Headers:")
-- for k, v in req.header_items():
-- log.debug(" %s: %s" % (k, v))
--
-- try:
-- res = urllib.request.urlopen(req, **kwargs)
--
-- except urllib.error.HTTPError as e:
-- # Log response headers
-- log.debug("HTTP Response: %s" % e.code)
-- log.debug(" Headers:")
-- for header in e.headers:
-- log.debug(" %s: %s" % (header, e.headers[header]))
--
-- # Raise all other errors
-- raise e
--
-- # Log response headers
-- log.debug("HTTP Response: %s" % res.code)
-- log.debug(" Headers:")
-- for k, v in res.getheaders():
-- log.debug(" %s: %s" % (k, v))
--
-- return res
--
-- def download(self, public_key, timestamp=None, tmpdir=None, **kwargs):
-- url = "%s/%s" % (self.version, DATABASE_FILENAME)
--
-- headers = {}
-- if timestamp:
-- headers["If-Modified-Since"] = time.strftime(
-- "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
-- )
--
-- t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
-- with t:
-- # Try all mirrors
-- for mirror in self.mirrors:
-- # Prepare HTTP request
-- req = self._make_request(url, baseurl=mirror, headers=headers)
--
-- try:
-- with self._send_request(req) as res:
-- decompressor = lzma.LZMADecompressor()
--
-- # Read all data
-- while True:
-- buf = res.read(1024)
-- if not buf:
-- break
--
-- # Decompress data
-- buf = decompressor.decompress(buf)
-- if buf:
-- t.write(buf)
--
-- # Write all data to disk
-- t.flush()
--
-- # Catch decompression errors
-- except lzma.LZMAError as e:
-- log.warning("Could not decompress downloaded file: %s" % e)
-- continue
--
-- except urllib.error.HTTPError as e:
-- # The file on the server was too old
-- if e.code == 304:
-- log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)
--
-- # Log any other HTTP errors
-- else:
-- log.warning("%s reported: %s" % (mirror, e))
--
-- # Throw away any downloaded content and try again
-- t.truncate()
--
-- else:
-- # Check if the downloaded database is recent
-- if not self._check_database(t, public_key, timestamp):
-- log.warning("Downloaded database is outdated. Trying next mirror...")
--
-- # Throw away the data and try again
-- t.truncate()
-- continue
--
-- # Make the file readable for everyone
-- os.chmod(t.name, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)
--
-- # Return temporary file
-- return t
--
-- # Delete the temporary file after unsuccessful downloads
-- os.unlink(t.name)
--
-- raise FileNotFoundError(url)
--
-- def _check_database(self, f, public_key, timestamp=None):
-- """
-- Checks the downloaded database if it can be opened,
-- verified and if it is recent enough
-- """
-- log.debug("Opening downloaded database at %s" % f.name)
--
-- db = Database(f.name)
--
-- # Database is not recent
-- if timestamp and db.created_at < timestamp:
-- return False
--
-- log.info("Downloaded new database from %s" % (time.strftime(
-- "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
-- )))
--
-- # Verify the database
-- with open(public_key, "r") as f:
-- if not db.verify(f):
-- log.error("Could not verify database")
-- return False
--
-- return True
-diff --git a/src/python/export.py b/src/python/export.py
-deleted file mode 100644
-index 942700e..0000000
---- a/src/python/export.py
-+++ /dev/null
-@@ -1,291 +0,0 @@
--#!/usr/bin/python3
--###############################################################################
--# #
--# libloc - A library to determine the location of someone on the Internet #
--# #
--# Copyright (C) 2020-2021 IPFire Development Team <info@ipfire.org> #
--# #
--# This library is free software; you can redistribute it and/or #
--# modify it under the terms of the GNU Lesser General Public #
--# License as published by the Free Software Foundation; either #
--# version 2.1 of the License, or (at your option) any later version. #
--# #
--# This library is distributed in the hope that it will be useful, #
--# but WITHOUT ANY WARRANTY; without even the implied warranty of #
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
--# Lesser General Public License for more details. #
--# #
--###############################################################################
--
--import io
--import ipaddress
--import logging
--import math
--import os
--import socket
--import sys
--
--from .i18n import _
--import _location
--
--# Initialise logging
--log = logging.getLogger("location.export")
--log.propagate = 1
--
--FLAGS = {
-- _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
-- _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
-- _location.NETWORK_FLAG_ANYCAST : "A3",
-- _location.NETWORK_FLAG_DROP : "XD",
--}
--
--class OutputWriter(object):
-- suffix = "networks"
-- mode = "w"
--
-- def __init__(self, name, family=None, directory=None, f=None):
-- self.name = name
-- self.family = family
-- self.directory = directory
--
-- # Open output file
-- if f:
-- self.f = f
-- elif self.directory:
-- self.f = open(self.filename, self.mode)
-- elif "b" in self.mode:
-- self.f = io.BytesIO()
-- else:
-- self.f = io.StringIO()
--
-- # Tag
-- self.tag = self._make_tag()
--
-- # Call any custom initialization
-- self.init()
--
-- # Immediately write the header
-- self._write_header()
--
-- def init(self):
-- """
-- To be overwritten by anything that inherits from this
-- """
-- pass
--
-- def __repr__(self):
-- return "<%s %s f=%s>" % (self.__class__.__name__, self, self.f)
--
-- def _make_tag(self):
-- families = {
-- socket.AF_INET6 : "6",
-- socket.AF_INET : "4",
-- }
--
-- return "%sv%s" % (self.name, families.get(self.family, "?"))
--
-- def filename(self):
-- if self.directory:
-- return os.path.join(self.directory, "%s.%s" % (self.tag, self.suffix))
--
-- def _write_header(self):
-- """
-- The header of the file
-- """
-- pass
--
-- def _write_footer(self):
-- """
-- The footer of the file
-- """
-- pass
--
-- def write(self, network):
-- self.f.write("%s\n" % network)
--
-- def finish(self):
-- """
-- Called when all data has been written
-- """
-- self._write_footer()
--
-- # Flush all output
-- self.f.flush()
--
-- def print(self):
-- """
-- Prints the entire output line by line
-- """
-- if isinstance(self.f, io.BytesIO):
-- raise TypeError(_("Won't write binary output to stdout"))
--
-- # Go back to the beginning
-- self.f.seek(0)
--
-- # Iterate over everything line by line
-- for line in self.f:
-- sys.stdout.write(line)
--
--
--class IpsetOutputWriter(OutputWriter):
-- """
-- For ipset
-- """
-- suffix = "ipset"
--
-- # The value is being used if we don't know any better
-- DEFAULT_HASHSIZE = 64
--
-- # We aim for this many networks in a bucket on average. This allows us to choose
-- # how much memory we want to sacrifice to gain better performance. The lower the
-- # factor, the faster a lookup will be, but it will use more memory.
-- # We will aim for only using three quarters of all buckets to avoid any searches
-- # through the linked lists.
-- HASHSIZE_FACTOR = 0.75
--
-- def init(self):
-- # Count all networks
-- self.networks = 0
--
-- @property
-- def hashsize(self):
-- """
-- Calculates an optimized hashsize
-- """
-- # Return the default value if we don't know the size of the set
-- if not self.networks:
-- return self.DEFAULT_HASHSIZE
--
-- # Find the nearest power of two that is larger than the number of networks
-- # divided by the hashsize factor.
-- exponent = math.log(self.networks / self.HASHSIZE_FACTOR, 2)
--
-- # Return the size of the hash (the minimum is 64)
-- return max(2 ** math.ceil(exponent), 64)
--
-- def _write_header(self):
-- # This must have a fixed size, because we will write the header again in the end
-- self.f.write("create %s hash:net family inet%s" % (
-- self.tag,
-- "6" if self.family == socket.AF_INET6 else ""
-- ))
-- self.f.write(" hashsize %8d maxelem 1048576 -exist\n" % self.hashsize)
-- self.f.write("flush %s\n" % self.tag)
--
-- def write(self, network):
-- self.f.write("add %s %s\n" % (self.tag, network))
--
-- # Increment network counter
-- self.networks += 1
--
-- def _write_footer(self):
-- # Jump back to the beginning of the file
-- self.f.seek(0)
--
-- # Rewrite the header with better configuration
-- self._write_header()
--
--
--class NftablesOutputWriter(OutputWriter):
-- """
-- For nftables
-- """
-- suffix = "set"
--
-- def _write_header(self):
-- self.f.write("define %s = {\n" % self.tag)
--
-- def _write_footer(self):
-- self.f.write("}\n")
--
-- def write(self, network):
-- self.f.write(" %s,\n" % network)
--
--
--class XTGeoIPOutputWriter(OutputWriter):
-- """
-- Formats the output in that way, that it can be loaded by
-- the xt_geoip kernel module from xtables-addons.
-- """
-- mode = "wb"
--
-- @property
-- def tag(self):
-- return self.name
--
-- @property
-- def suffix(self):
-- return "iv%s" % ("6" if self.family == socket.AF_INET6 else "4")
--
-- def write(self, network):
-- self.f.write(network._first_address)
-- self.f.write(network._last_address)
--
--
--formats = {
-- "ipset" : IpsetOutputWriter,
-- "list" : OutputWriter,
-- "nftables" : NftablesOutputWriter,
-- "xt_geoip" : XTGeoIPOutputWriter,
--}
--
--class Exporter(object):
-- def __init__(self, db, writer):
-- self.db, self.writer = db, writer
--
-- def export(self, directory, families, countries, asns):
-- for family in families:
-- log.debug("Exporting family %s" % family)
--
-- writers = {}
--
-- # Create writers for countries
-- for country_code in countries:
-- writers[country_code] = self.writer(country_code, family=family, directory=directory)
--
-- # Create writers for ASNs
-- for asn in asns:
-- writers[asn] = self.writer("AS%s" % asn, family=family, directory=directory)
--
-- # Filter countries from special country codes
-- country_codes = [
-- country_code for country_code in countries if not country_code in FLAGS.values()
-- ]
--
-- # Get all networks that match the family
-- networks = self.db.search_networks(family=family,
-- country_codes=country_codes, asns=asns, flatten=True)
--
-- # Walk through all networks
-- for network in networks:
-- # Write matching countries
-- try:
-- writers[network.country_code].write(network)
-- except KeyError:
-- pass
--
-- # Write matching ASNs
-- try:
-- writers[network.asn].write(network)
-- except KeyError:
-- pass
--
-- # Handle flags
-- for flag in FLAGS:
-- if network.has_flag(flag):
-- # Fetch the "fake" country code
-- country = FLAGS[flag]
--
-- try:
-- writers[country].write(network)
-- except KeyError:
-- pass
--
-- # Write everything to the filesystem
-- for writer in writers.values():
-- writer.finish()
--
-- # Print to stdout
-- if not directory:
-- for writer in writers.values():
-- writer.print()
-diff --git a/src/python/i18n.py b/src/python/i18n.py
-deleted file mode 100644
-index 2161aa6..0000000
---- a/src/python/i18n.py
-+++ /dev/null
-@@ -1,26 +0,0 @@
--#!/usr/bin/python3
--###############################################################################
--# #
--# libloc - A library to determine the location of someone on the Internet #
--# #
--# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
--# #
--# This library is free software; you can redistribute it and/or #
--# modify it under the terms of the GNU Lesser General Public #
--# License as published by the Free Software Foundation; either #
--# version 2.1 of the License, or (at your option) any later version. #
--# #
--# This library is distributed in the hope that it will be useful, #
--# but WITHOUT ANY WARRANTY; without even the implied warranty of #
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
--# Lesser General Public License for more details. #
--# #
--###############################################################################
--
--import gettext
--
--def _(singular, plural=None, n=None):
-- if plural:
-- return gettext.dngettext("libloc", singular, plural, n)
--
-- return gettext.dgettext("libloc", singular)
-diff --git a/src/python/importer.py b/src/python/importer.py
-deleted file mode 100644
-index dee36ed..0000000
---- a/src/python/importer.py
-+++ /dev/null
-@@ -1,250 +0,0 @@
--#!/usr/bin/python3
--###############################################################################
--# #
--# libloc - A library to determine the location of someone on the Internet #
--# #
--# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
--# #
--# This library is free software; you can redistribute it and/or #
--# modify it under the terms of the GNU Lesser General Public #
--# License as published by the Free Software Foundation; either #
--# version 2.1 of the License, or (at your option) any later version. #
--# #
--# This library is distributed in the hope that it will be useful, #
--# but WITHOUT ANY WARRANTY; without even the implied warranty of #
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
--# Lesser General Public License for more details. #
--# #
--###############################################################################
--
--import gzip
--import logging
--import urllib.request
--
--# Initialise logging
--log = logging.getLogger("location.importer")
--log.propagate = 1
--
--WHOIS_SOURCES = {
-- # African Network Information Centre
-- "AFRINIC": [
-- "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz"
-- ],
--
-- # Asia Pacific Network Information Centre
-- "APNIC": [
-- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
-- #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
-- #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
-- "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
-- "https://ftp.apnic.net/apnic/whois/apnic.db.organisation.gz"
-- ],
--
-- # American Registry for Internet Numbers
-- # XXX there is nothing useful for us in here
-- # ARIN: [
-- # "https://ftp.arin.net/pub/rr/arin.db"
-- # ],
--
-- # Japan Network Information Center
-- "JPNIC": [
-- "https://ftp.nic.ad.jp/jpirr/jpirr.db.gz"
-- ],
--
-- # Latin America and Caribbean Network Information Centre
-- "LACNIC": [
-- "https://ftp.lacnic.net/lacnic/dbase/lacnic.db.gz"
-- ],
--
-- # Réseaux IP Européens
-- "RIPE": [
-- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
-- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
-- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
-- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
-- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.organisation.gz"
-- ],
--}
--
--EXTENDED_SOURCES = {
-- # African Network Information Centre
-- # "ARIN": [
-- # "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest"
-- # ],
--
-- # Asia Pacific Network Information Centre
-- # "APNIC": [
-- # "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest"
-- # ],
--
-- # American Registry for Internet Numbers
-- "ARIN": [
-- "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest"
-- ],
--
-- # Latin America and Caribbean Network Information Centre
-- "LACNIC": [
-- "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest"
-- ],
--
-- # Réseaux IP Européens
-- # "RIPE": [
-- # "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest"
-- # ],
--}
--
--class Downloader(object):
-- def __init__(self):
-- self.proxy = None
--
-- def set_proxy(self, url):
-- """
-- Sets a HTTP proxy that is used to perform all requests
-- """
-- log.info("Using proxy %s" % url)
-- self.proxy = url
--
-- def request(self, url, data=None, return_blocks=False):
-- req = urllib.request.Request(url, data=data)
--
-- # Configure proxy
-- if self.proxy:
-- req.set_proxy(self.proxy, "http")
--
-- return DownloaderContext(self, req, return_blocks=return_blocks)
--
--
--class DownloaderContext(object):
-- def __init__(self, downloader, request, return_blocks=False):
-- self.downloader = downloader
-- self.request = request
--
-- # Should we return one block or a single line?
-- self.return_blocks = return_blocks
--
-- # Save the response object
-- self.response = None
--
-- def __enter__(self):
-- log.info("Retrieving %s..." % self.request.full_url)
--
-- # Send request
-- self.response = urllib.request.urlopen(self.request)
--
-- # Log the response headers
-- log.debug("Response Headers:")
-- for header in self.headers:
-- log.debug(" %s: %s" % (header, self.get_header(header)))
--
-- return self
--
-- def __exit__(self, type, value, traceback):
-- pass
--
-- def __iter__(self):
-- """
-- Makes the object iterable by going through each block
-- """
-- if self.return_blocks:
-- return iterate_over_blocks(self.body)
--
-- return iterate_over_lines(self.body)
--
-- @property
-- def headers(self):
-- if self.response:
-- return self.response.headers
--
-- def get_header(self, name):
-- if self.headers:
-- return self.headers.get(name)
--
-- @property
-- def body(self):
-- """
-- Returns a file-like object with the decoded content
-- of the response.
-- """
-- content_type = self.get_header("Content-Type")
--
-- # Decompress any gzipped response on the fly
-- if content_type in ("application/x-gzip", "application/gzip"):
-- return gzip.GzipFile(fileobj=self.response, mode="rb")
--
-- # Return the response by default
-- return self.response
--
--
--def read_blocks(f):
-- for block in iterate_over_blocks(f):
-- type = None
-- data = {}
--
-- for i, line in enumerate(block):
-- key, value = line.split(":", 1)
--
-- # The key of the first line defines the type
-- if i == 0:
-- type = key
--
-- # Store value
-- data[key] = value.strip()
--
-- yield type, data
--
--def iterate_over_blocks(f, charsets=("utf-8", "latin1")):
-- block = []
--
-- for line in f:
-- # Convert to string
-- for charset in charsets:
-- try:
-- line = line.decode(charset)
-- except UnicodeDecodeError:
-- continue
-- else:
-- break
--
-- # Skip commented lines
-- if line.startswith("#") or line.startswith("%"):
-- continue
--
-- # Strip line-endings
-- line = line.rstrip()
--
-- # Remove any comments at the end of line
-- line, hash, comment = line.partition("#")
--
-- if comment:
-- # Strip any whitespace before the comment
-- line = line.rstrip()
--
-- # If the line is now empty, we move on
-- if not line:
-- continue
--
-- if line:
-- block.append(line)
-- continue
--
-- # End the block on an empty line
-- if block:
-- yield block
--
-- # Reset the block
-- block = []
--
-- # Return the last block
-- if block:
-- yield block
--
--
--def iterate_over_lines(f):
-- for line in f:
-- # Decode the line
-- line = line.decode()
--
-- # Strip the ending
-- yield line.rstrip()
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-deleted file mode 100644
-index bee9186..0000000
---- a/src/python/location-importer.in
-+++ /dev/null
-@@ -1,1535 +0,0 @@
--#!/usr/bin/python3
--###############################################################################
--# #
--# libloc - A library to determine the location of someone on the Internet #
--# #
--# Copyright (C) 2020-2022 IPFire Development Team <info@ipfire.org> #
--# #
--# This library is free software; you can redistribute it and/or #
--# modify it under the terms of the GNU Lesser General Public #
--# License as published by the Free Software Foundation; either #
--# version 2.1 of the License, or (at your option) any later version. #
--# #
--# This library is distributed in the hope that it will be useful, #
--# but WITHOUT ANY WARRANTY; without even the implied warranty of #
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
--# Lesser General Public License for more details. #
--# #
--###############################################################################
--
--import argparse
--import ipaddress
--import json
--import logging
--import math
--import re
--import socket
--import sys
--import telnetlib
--
--# Load our location module
--import location
--import location.database
--import location.importer
--from location.i18n import _
--
--# Initialise logging
--log = logging.getLogger("location.importer")
--log.propagate = 1
--
--# Define constants
--VALID_ASN_RANGES = (
-- (1, 23455),
-- (23457, 64495),
-- (131072, 4199999999),
--)
--
--
--class CLI(object):
-- def parse_cli(self):
-- parser = argparse.ArgumentParser(
-- description=_("Location Importer Command Line Interface"),
-- )
-- subparsers = parser.add_subparsers()
--
-- # Global configuration flags
-- parser.add_argument("--debug", action="store_true",
-- help=_("Enable debug output"))
-- parser.add_argument("--quiet", action="store_true",
-- help=_("Enable quiet mode"))
--
-- # version
-- parser.add_argument("--version", action="version",
-- version="%(prog)s @VERSION@")
--
-- # Database
-- parser.add_argument("--database-host", required=True,
-- help=_("Database Hostname"), metavar=_("HOST"))
-- parser.add_argument("--database-name", required=True,
-- help=_("Database Name"), metavar=_("NAME"))
-- parser.add_argument("--database-username", required=True,
-- help=_("Database Username"), metavar=_("USERNAME"))
-- parser.add_argument("--database-password", required=True,
-- help=_("Database Password"), metavar=_("PASSWORD"))
--
-- # Write Database
-- write = subparsers.add_parser("write", help=_("Write database to file"))
-- write.set_defaults(func=self.handle_write)
-- write.add_argument("file", nargs=1, help=_("Database File"))
-- write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
-- write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
-- write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
-- write.add_argument("--description", nargs="?", help=_("Sets a description"))
-- write.add_argument("--license", nargs="?", help=_("Sets the license"))
-- write.add_argument("--version", type=int, help=_("Database Format Version"))
--
-- # Update WHOIS
-- update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
-- update_whois.set_defaults(func=self.handle_update_whois)
--
-- # Update announcements
-- update_announcements = subparsers.add_parser("update-announcements",
-- help=_("Update BGP Annoucements"))
-- update_announcements.set_defaults(func=self.handle_update_announcements)
-- update_announcements.add_argument("server", nargs=1,
-- help=_("Route Server to connect to"), metavar=_("SERVER"))
--
-- # Update overrides
-- update_overrides = subparsers.add_parser("update-overrides",
-- help=_("Update overrides"),
-- )
-- update_overrides.add_argument(
-- "files", nargs="+", help=_("Files to import"),
-- )
-- update_overrides.set_defaults(func=self.handle_update_overrides)
--
-- # Import countries
-- import_countries = subparsers.add_parser("import-countries",
-- help=_("Import countries"),
-- )
-- import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
-- help=_("File to import"))
-- import_countries.set_defaults(func=self.handle_import_countries)
--
-- args = parser.parse_args()
--
-- # Configure logging
-- if args.debug:
-- location.logger.set_level(logging.DEBUG)
-- elif args.quiet:
-- location.logger.set_level(logging.WARNING)
--
-- # Print usage if no action was given
-- if not "func" in args:
-- parser.print_usage()
-- sys.exit(2)
--
-- return args
--
-- def run(self):
-- # Parse command line arguments
-- args = self.parse_cli()
--
-- # Initialise database
-- self.db = self._setup_database(args)
--
-- # Call function
-- ret = args.func(args)
--
-- # Return with exit code
-- if ret:
-- sys.exit(ret)
--
-- # Otherwise just exit
-- sys.exit(0)
--
-- def _setup_database(self, ns):
-- """
-- Initialise the database
-- """
-- # Connect to database
-- db = location.database.Connection(
-- host=ns.database_host, database=ns.database_name,
-- user=ns.database_username, password=ns.database_password,
-- )
--
-- with db.transaction():
-- db.execute("""
-- -- announcements
-- CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
-- first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
-- last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
-- CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
-- CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
-- CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
--
-- -- autnums
-- CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
-- ALTER TABLE autnums ADD COLUMN IF NOT EXISTS source text;
-- CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
--
-- -- countries
-- CREATE TABLE IF NOT EXISTS countries(
-- country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
-- CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
--
-- -- networks
-- CREATE TABLE IF NOT EXISTS networks(network inet, country text);
-- ALTER TABLE networks ADD COLUMN IF NOT EXISTS original_countries text[];
-- ALTER TABLE networks ADD COLUMN IF NOT EXISTS source text;
-- CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
-- CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
-- CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
--
-- -- overrides
-- CREATE TABLE IF NOT EXISTS autnum_overrides(
-- number bigint NOT NULL,
-- name text,
-- country text,
-- is_anonymous_proxy boolean,
-- is_satellite_provider boolean,
-- is_anycast boolean
-- );
-- CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
-- ON autnum_overrides(number);
-- ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS source text;
-- ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
--
-- CREATE TABLE IF NOT EXISTS network_overrides(
-- network inet NOT NULL,
-- country text,
-- is_anonymous_proxy boolean,
-- is_satellite_provider boolean,
-- is_anycast boolean
-- );
-- CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
-- ON network_overrides(network);
-- CREATE INDEX IF NOT EXISTS network_overrides_search
-- ON network_overrides USING GIST(network inet_ops);
-- ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS source text;
-- ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
-- """)
--
-- return db
--
-- def handle_write(self, ns):
-- """
-- Compiles a database in libloc format out of what is in the database
-- """
-- # Allocate a writer
-- writer = location.Writer(ns.signing_key, ns.backup_signing_key)
--
-- # Set all metadata
-- if ns.vendor:
-- writer.vendor = ns.vendor
--
-- if ns.description:
-- writer.description = ns.description
--
-- if ns.license:
-- writer.license = ns.license
--
-- # Add all Autonomous Systems
-- log.info("Writing Autonomous Systems...")
--
-- # Select all ASes with a name
-- rows = self.db.query("""
-- SELECT
-- autnums.number AS number,
-- COALESCE(
-- (SELECT overrides.name FROM autnum_overrides overrides
-- WHERE overrides.number = autnums.number),
-- autnums.name
-- ) AS name
-- FROM autnums
-- WHERE name <> %s ORDER BY number
-- """, "")
--
-- for row in rows:
-- a = writer.add_as(row.number)
-- a.name = row.name
--
-- # Add all networks
-- log.info("Writing networks...")
--
-- # Select all known networks
-- rows = self.db.query("""
-- WITH known_networks AS (
-- SELECT network FROM announcements
-- UNION
-- SELECT network FROM networks
-- UNION
-- SELECT network FROM network_overrides
-- ),
--
-- ordered_networks AS (
-- SELECT
-- known_networks.network AS network,
-- announcements.autnum AS autnum,
-- networks.country AS country,
--
-- -- Must be part of returned values for ORDER BY clause
-- masklen(announcements.network) AS sort_a,
-- masklen(networks.network) AS sort_b
-- FROM
-- known_networks
-- LEFT JOIN
-- announcements ON known_networks.network <<= announcements.network
-- LEFT JOIN
-- networks ON known_networks.network <<= networks.network
-- ORDER BY
-- known_networks.network,
-- sort_a DESC,
-- sort_b DESC
-- )
--
-- -- Return a list of those networks enriched with all
-- -- other information that we store in the database
-- SELECT
-- DISTINCT ON (network)
-- network,
-- autnum,
--
-- -- Country
-- COALESCE(
-- (
-- SELECT country FROM network_overrides overrides
-- WHERE networks.network <<= overrides.network
-- ORDER BY masklen(overrides.network) DESC
-- LIMIT 1
-- ),
-- (
-- SELECT country FROM autnum_overrides overrides
-- WHERE networks.autnum = overrides.number
-- ),
-- networks.country
-- ) AS country,
--
-- -- Flags
-- COALESCE(
-- (
-- SELECT is_anonymous_proxy FROM network_overrides overrides
-- WHERE networks.network <<= overrides.network
-- ORDER BY masklen(overrides.network) DESC
-- LIMIT 1
-- ),
-- (
-- SELECT is_anonymous_proxy FROM autnum_overrides overrides
-- WHERE networks.autnum = overrides.number
-- ),
-- FALSE
-- ) AS is_anonymous_proxy,
-- COALESCE(
-- (
-- SELECT is_satellite_provider FROM network_overrides overrides
-- WHERE networks.network <<= overrides.network
-- ORDER BY masklen(overrides.network) DESC
-- LIMIT 1
-- ),
-- (
-- SELECT is_satellite_provider FROM autnum_overrides overrides
-- WHERE networks.autnum = overrides.number
-- ),
-- FALSE
-- ) AS is_satellite_provider,
-- COALESCE(
-- (
-- SELECT is_anycast FROM network_overrides overrides
-- WHERE networks.network <<= overrides.network
-- ORDER BY masklen(overrides.network) DESC
-- LIMIT 1
-- ),
-- (
-- SELECT is_anycast FROM autnum_overrides overrides
-- WHERE networks.autnum = overrides.number
-- ),
-- FALSE
-- ) AS is_anycast,
-- COALESCE(
-- (
-- SELECT is_drop FROM network_overrides overrides
-- WHERE networks.network <<= overrides.network
-- ORDER BY masklen(overrides.network) DESC
-- LIMIT 1
-- ),
-- (
-- SELECT is_drop FROM autnum_overrides overrides
-- WHERE networks.autnum = overrides.number
-- ),
-- FALSE
-- ) AS is_drop
-- FROM
-- ordered_networks networks
-- """)
--
-- for row in rows:
-- network = writer.add_network(row.network)
--
-- # Save country
-- if row.country:
-- network.country_code = row.country
--
-- # Save ASN
-- if row.autnum:
-- network.asn = row.autnum
--
-- # Set flags
-- if row.is_anonymous_proxy:
-- network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
--
-- if row.is_satellite_provider:
-- network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
--
-- if row.is_anycast:
-- network.set_flag(location.NETWORK_FLAG_ANYCAST)
--
-- if row.is_drop:
-- network.set_flag(location.NETWORK_FLAG_DROP)
--
-- # Add all countries
-- log.info("Writing countries...")
-- rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
--
-- for row in rows:
-- c = writer.add_country(row.country_code)
-- c.continent_code = row.continent_code
-- c.name = row.name
--
-- # Write everything to file
-- log.info("Writing database to file...")
-- for file in ns.file:
-- writer.write(file)
--
-- def handle_update_whois(self, ns):
-- downloader = location.importer.Downloader()
--
-- # Download all sources
-- with self.db.transaction():
-- # Create some temporary tables to store parsed data
-- self.db.execute("""
-- CREATE TEMPORARY TABLE _autnums(number integer NOT NULL, organization text NOT NULL, source text NOT NULL)
-- ON COMMIT DROP;
-- CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
--
-- CREATE TEMPORARY TABLE _organizations(handle text NOT NULL, name text NOT NULL, source text NOT NULL)
-- ON COMMIT DROP;
-- CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
--
-- CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL, original_countries text[] NOT NULL, source text NOT NULL)
-- ON COMMIT DROP;
-- CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
-- CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
-- """)
--
-- # Remove all previously imported content
-- self.db.execute("""
-- TRUNCATE TABLE networks;
-- """)
--
-- # Fetch all valid country codes to check parsed networks aganist...
-- rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
-- validcountries = []
--
-- for row in rows:
-- validcountries.append(row.country_code)
--
-- for source_key in location.importer.WHOIS_SOURCES:
-- for single_url in location.importer.WHOIS_SOURCES[source_key]:
-- with downloader.request(single_url, return_blocks=True) as f:
-- for block in f:
-- self._parse_block(block, source_key, validcountries)
--
-- # Process all parsed networks from every RIR we happen to have access to,
-- # insert the largest network chunks into the networks table immediately...
-- families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
--
-- for family in (row.family for row in families):
-- smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
--
-- self.db.execute("INSERT INTO networks(network, country, original_countries, source) \
-- SELECT network, country, original_countries, source FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
--
-- # ... determine any other prefixes for this network family, ...
-- prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
-- WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
--
-- # ... and insert networks with this prefix in case they provide additional
-- # information (i. e. subnet of a larger chunk with a different country)
-- for prefix in (row.prefix for row in prefixes):
-- self.db.execute("""
-- WITH candidates AS (
-- SELECT
-- _rirdata.network,
-- _rirdata.country,
-- _rirdata.original_countries,
-- _rirdata.source
-- FROM
-- _rirdata
-- WHERE
-- family(_rirdata.network) = %s
-- AND
-- masklen(_rirdata.network) = %s
-- ),
-- filtered AS (
-- SELECT
-- DISTINCT ON (c.network)
-- c.network,
-- c.country,
-- c.original_countries,
-- c.source,
-- masklen(networks.network),
-- networks.country AS parent_country
-- FROM
-- candidates c
-- LEFT JOIN
-- networks
-- ON
-- c.network << networks.network
-- ORDER BY
-- c.network,
-- masklen(networks.network) DESC NULLS LAST
-- )
-- INSERT INTO
-- networks(network, country, original_countries, source)
-- SELECT
-- network,
-- country,
-- original_countries,
-- source
-- FROM
-- filtered
-- WHERE
-- parent_country IS NULL
-- OR
-- country <> parent_country
-- ON CONFLICT DO NOTHING""",
-- family, prefix,
-- )
--
-- self.db.execute("""
-- INSERT INTO autnums(number, name, source)
-- SELECT _autnums.number, _organizations.name, _organizations.source FROM _autnums
-- JOIN _organizations ON _autnums.organization = _organizations.handle
-- ON CONFLICT (number) DO UPDATE SET name = excluded.name;
-- """)
--
-- # Download all extended sources
-- for source_key in location.importer.EXTENDED_SOURCES:
-- for single_url in location.importer.EXTENDED_SOURCES[source_key]:
-- with self.db.transaction():
-- # Download data
-- with downloader.request(single_url) as f:
-- for line in f:
-- self._parse_line(line, source_key, validcountries)
--
-- # Download and import (technical) AS names from ARIN
-- self._import_as_names_from_arin()
--
-- def _check_parsed_network(self, network):
-- """
-- Assistive function to detect and subsequently sort out parsed
-- networks from RIR data (both Whois and so-called "extended sources"),
-- which are or have...
--
-- (a) not globally routable (RFC 1918 space, et al.)
-- (b) covering a too large chunk of the IP address space (prefix length
-- is < 7 for IPv4 networks, and < 10 for IPv6)
-- (c) "0.0.0.0" or "::" as a network address
-- (d) are too small for being publicly announced (we have decided not to
-- process them at the moment, as they significantly enlarge our
-- database without providing very helpful additional information)
--
-- This unfortunately is necessary due to brain-dead clutter across
-- various RIR databases, causing mismatches and eventually disruptions.
--
-- We will return False in case a network is not suitable for adding
-- it to our database, and True otherwise.
-- """
--
-- if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
-- return False
--
-- if not network.is_global:
-- log.debug("Skipping non-globally routable network: %s" % network)
-- return False
--
-- if network.version == 4:
-- if network.prefixlen < 7:
-- log.debug("Skipping too big IP chunk: %s" % network)
-- return False
--
-- if network.prefixlen > 24:
-- log.debug("Skipping network too small to be publicly announced: %s" % network)
-- return False
--
-- if str(network.network_address) == "0.0.0.0":
-- log.debug("Skipping network based on 0.0.0.0: %s" % network)
-- return False
--
-- elif network.version == 6:
-- if network.prefixlen < 10:
-- log.debug("Skipping too big IP chunk: %s" % network)
-- return False
--
-- if network.prefixlen > 48:
-- log.debug("Skipping network too small to be publicly announced: %s" % network)
-- return False
--
-- if str(network.network_address) == "::":
-- log.debug("Skipping network based on '::': %s" % network)
-- return False
--
-- else:
-- # This should not happen...
-- log.warning("Skipping network of unknown family, this should not happen: %s" % network)
-- return False
--
-- # In case we have made it here, the network is considered to
-- # be suitable for libloc consumption...
-- return True
--
-- def _check_parsed_asn(self, asn):
-- """
-- Assistive function to filter Autonomous System Numbers not being suitable
-- for adding to our database. Returns False in such cases, and True otherwise.
-- """
--
-- for start, end in VALID_ASN_RANGES:
-- if start <= asn and end >= asn:
-- return True
--
-- log.info("Supplied ASN %s out of publicly routable ASN ranges" % asn)
-- return False
--
-- def _parse_block(self, block, source_key, validcountries = None):
-- # Get first line to find out what type of block this is
-- line = block[0]
--
-- # aut-num
-- if line.startswith("aut-num:"):
-- return self._parse_autnum_block(block, source_key)
--
-- # inetnum
-- if line.startswith("inet6num:") or line.startswith("inetnum:"):
-- return self._parse_inetnum_block(block, source_key, validcountries)
--
-- # organisation
-- elif line.startswith("organisation:"):
-- return self._parse_org_block(block, source_key)
--
-- def _parse_autnum_block(self, block, source_key):
-- autnum = {}
-- for line in block:
-- # Split line
-- key, val = split_line(line)
--
-- if key == "aut-num":
-- m = re.match(r"^(AS|as)(\d+)", val)
-- if m:
-- autnum["asn"] = m.group(2)
--
-- elif key == "org":
-- autnum[key] = val.upper()
--
-- elif key == "descr":
-- # Save the first description line as well...
-- if not key in autnum:
-- autnum[key] = val
--
-- # Skip empty objects
-- if not autnum or not "asn" in autnum:
-- return
--
-- # Insert a dummy organisation handle into our temporary organisations
-- # table in case the AS does not have an organisation handle set, but
-- # has a description (a quirk often observed in APNIC area), so we can
-- # later display at least some string for this AS.
-- if not "org" in autnum:
-- if "descr" in autnum:
-- autnum["org"] = "LIBLOC-%s-ORGHANDLE" % autnum.get("asn")
--
-- self.db.execute("INSERT INTO _organizations(handle, name, source) \
-- VALUES(%s, %s, %s) ON CONFLICT (handle) DO NOTHING",
-- autnum.get("org"), autnum.get("descr"), source_key,
-- )
-- else:
-- log.warning("ASN %s neither has an organisation handle nor a description line set, omitting" % \
-- autnum.get("asn"))
-- return
--
-- # Insert into database
-- self.db.execute("INSERT INTO _autnums(number, organization, source) \
-- VALUES(%s, %s, %s) ON CONFLICT (number) DO UPDATE SET \
-- organization = excluded.organization",
-- autnum.get("asn"), autnum.get("org"), source_key,
-- )
--
-- def _parse_inetnum_block(self, block, source_key, validcountries = None):
-- log.debug("Parsing inetnum block:")
--
-- inetnum = {}
-- for line in block:
-- log.debug(line)
--
-- # Split line
-- key, val = split_line(line)
--
-- # Filter any inetnum records which are only referring to IP space
-- # not managed by that specific RIR...
-- if key == "netname":
-- if re.match(r"^(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK|STUB-[\d-]{3,}SLASH\d{1,2})", val.strip()):
-- log.debug("Skipping record indicating historic/orphaned data: %s" % val.strip())
-- return
--
-- if key == "inetnum":
-- start_address, delim, end_address = val.partition("-")
--
-- # Strip any excess space
-- start_address, end_address = start_address.rstrip(), end_address.strip()
--
-- # Handle "inetnum" formatting in LACNIC DB (e.g. "24.152.8/22" instead of "24.152.8.0/22")
-- if start_address and not (delim or end_address):
-- try:
-- start_address = ipaddress.ip_network(start_address, strict=False)
-- except ValueError:
-- start_address = start_address.split("/")
-- ldigits = start_address[0].count(".")
--
-- # How many octets do we need to add?
-- # (LACNIC does not seem to have a /8 or greater assigned, so the following should suffice.)
-- if ldigits == 1:
-- start_address = start_address[0] + ".0.0/" + start_address[1]
-- elif ldigits == 2:
-- start_address = start_address[0] + ".0/" + start_address[1]
-- else:
-- log.warning("Could not recover IPv4 address from line in LACNIC DB format: %s" % line)
-- return
--
-- try:
-- start_address = ipaddress.ip_network(start_address, strict=False)
-- except ValueError:
-- log.warning("Could not parse line in LACNIC DB format: %s" % line)
-- return
--
-- # Enumerate first and last IP address of this network
-- end_address = start_address[-1]
-- start_address = start_address[0]
--
-- else:
-- # Convert to IP address
-- try:
-- start_address = ipaddress.ip_address(start_address)
-- end_address = ipaddress.ip_address(end_address)
-- except ValueError:
-- log.warning("Could not parse line: %s" % line)
-- return
--
-- inetnum["inetnum"] = list(ipaddress.summarize_address_range(start_address, end_address))
--
-- elif key == "inet6num":
-- inetnum[key] = [ipaddress.ip_network(val, strict=False)]
--
-- elif key == "country":
-- val = val.upper()
--
-- # Catch RIR data objects with more than one country code...
-- if not key in inetnum:
-- inetnum[key] = []
-- else:
-- if val in inetnum.get("country"):
-- # ... but keep this list distinct...
-- continue
--
-- # When people set country codes to "UK", they actually mean "GB"
-- if val == "UK":
-- val = "GB"
--
-- inetnum[key].append(val)
--
-- # Skip empty objects
-- if not inetnum or not "country" in inetnum:
-- return
--
-- # Prepare skipping objects with unknown country codes...
-- invalidcountries = [singlecountry for singlecountry in inetnum.get("country") if singlecountry not in validcountries]
--
-- # Iterate through all networks enumerated from above, check them for plausibility and insert
-- # them into the database, if _check_parsed_network() succeeded
-- for single_network in inetnum.get("inet6num") or inetnum.get("inetnum"):
-- if self._check_parsed_network(single_network):
--
-- # Skip objects with unknown country codes if they are valid to avoid log spam...
-- if validcountries and invalidcountries:
-- log.warning("Skipping network with bogus countr(y|ies) %s (original countries: %s): %s" % \
-- (invalidcountries, inetnum.get("country"), inetnum.get("inet6num") or inetnum.get("inetnum")))
-- break
--
-- # Everything is fine here, run INSERT statement...
-- self.db.execute("INSERT INTO _rirdata(network, country, original_countries, source) \
-- VALUES(%s, %s, %s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
-- "%s" % single_network, inetnum.get("country")[0], inetnum.get("country"), source_key,
-- )
--
-- def _parse_org_block(self, block, source_key):
-- org = {}
-- for line in block:
-- # Split line
-- key, val = split_line(line)
--
-- if key == "organisation":
-- org[key] = val.upper()
-- elif key == "org-name":
-- org[key] = val
--
-- # Skip empty objects
-- if not org:
-- return
--
-- self.db.execute("INSERT INTO _organizations(handle, name, source) \
-- VALUES(%s, %s, %s) ON CONFLICT (handle) DO \
-- UPDATE SET name = excluded.name",
-- org.get("organisation"), org.get("org-name"), source_key,
-- )
--
-- def _parse_line(self, line, source_key, validcountries = None):
-- # Skip version line
-- if line.startswith("2"):
-- return
--
-- # Skip comments
-- if line.startswith("#"):
-- return
--
-- try:
-- registry, country_code, type, line = line.split("|", 3)
-- except:
-- log.warning("Could not parse line: %s" % line)
-- return
--
-- # Skip any lines that are for stats only or do not have a country
-- # code at all (avoids log spam below)
-- if not country_code or country_code == '*':
-- return
--
-- # Skip objects with unknown country codes
-- if validcountries and country_code not in validcountries:
-- log.warning("Skipping line with bogus country '%s': %s" % \
-- (country_code, line))
-- return
--
-- if type in ("ipv6", "ipv4"):
-- return self._parse_ip_line(country_code, type, line, source_key)
--
-- def _parse_ip_line(self, country, type, line, source_key):
-- try:
-- address, prefix, date, status, organization = line.split("|")
-- except ValueError:
-- organization = None
--
-- # Try parsing the line without organization
-- try:
-- address, prefix, date, status = line.split("|")
-- except ValueError:
-- log.warning("Unhandled line format: %s" % line)
-- return
--
-- # Skip anything that isn't properly assigned
-- if not status in ("assigned", "allocated"):
-- return
--
-- # Cast prefix into an integer
-- try:
-- prefix = int(prefix)
-- except:
-- log.warning("Invalid prefix: %s" % prefix)
-- return
--
-- # Fix prefix length for IPv4
-- if type == "ipv4":
-- prefix = 32 - int(math.log(prefix, 2))
--
-- # Try to parse the address
-- try:
-- network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
-- except ValueError:
-- log.warning("Invalid IP address: %s" % address)
-- return
--
-- if not self._check_parsed_network(network):
-- return
--
-- self.db.execute("INSERT INTO networks(network, country, original_countries, source) \
-- VALUES(%s, %s, %s, %s) ON CONFLICT (network) DO \
-- UPDATE SET country = excluded.country",
-- "%s" % network, country, [country], source_key,
-- )
--
-- def _import_as_names_from_arin(self):
-- downloader = location.importer.Downloader()
--
-- # XXX: Download AS names file from ARIN (note that these names appear to be quite
-- # technical, not intended for human consumption, as description fields in
-- # organisation handles for other RIRs are - however, this is what we have got,
-- # and in some cases, it might be still better than nothing)
-- with downloader.request("https://ftp.arin.net/info/asn.txt", return_blocks=False) as f:
-- for line in f:
-- # Convert binary line to string...
-- line = str(line)
--
-- # ... valid lines start with a space, followed by the number of the Autonomous System ...
-- if not line.startswith(" "):
-- continue
--
-- # Split line and check if there is a valid ASN in it...
-- asn, name = line.split()[0:2]
--
-- try:
-- asn = int(asn)
-- except ValueError:
-- log.debug("Skipping ARIN AS names line not containing an integer for ASN")
-- continue
--
-- # Filter invalid ASNs...
-- if not self._check_parsed_asn(asn):
-- continue
--
-- # Skip any AS name that appears to be a placeholder for a different RIR or entity...
-- if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|DNIC|LACNIC|RIPE|IANA)(?:\d?$|\-)", name):
-- continue
--
-- # Bail out in case the AS name contains anything we do not expect here...
-- if re.search(r"[^a-zA-Z0-9-_]", name):
-- log.debug("Skipping ARIN AS name for %s containing invalid characters: %s" % \
-- (asn, name))
--
-- # Things look good here, run INSERT statement and skip this one if we already have
-- # a (better?) name for this Autonomous System...
-- self.db.execute("""
-- INSERT INTO autnums(
-- number,
-- name,
-- source
-- ) VALUES (%s, %s, %s)
-- ON CONFLICT (number) DO NOTHING""",
-- asn,
-- name,
-- "ARIN",
-- )
--
-- def handle_update_announcements(self, ns):
-- server = ns.server[0]
--
-- with self.db.transaction():
-- if server.startswith("/"):
-- self._handle_update_announcements_from_bird(server)
-- else:
-- self._handle_update_announcements_from_telnet(server)
--
-- # Purge anything we never want here
-- self.db.execute("""
-- -- Delete default routes
-- DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
--
-- -- Delete anything that is not global unicast address space
-- DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
--
-- -- DELETE "current network" address space
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
--
-- -- DELETE local loopback address space
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
--
-- -- DELETE RFC 1918 address space
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
--
-- -- DELETE test, benchmark and documentation address space
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
--
-- -- DELETE CGNAT address space (RFC 6598)
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
--
-- -- DELETE link local address space
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
--
-- -- DELETE IPv6 to IPv4 (6to4) address space (RFC 3068)
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
-- DELETE FROM announcements WHERE family(network) = 6 AND network <<= '2002::/16';
--
-- -- DELETE multicast and reserved address space
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
-- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
--
-- -- Delete networks that are too small to be in the global routing table
-- DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
-- DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
--
-- -- Delete any non-public or reserved ASNs
-- DELETE FROM announcements WHERE NOT (
-- (autnum >= 1 AND autnum <= 23455)
-- OR
-- (autnum >= 23457 AND autnum <= 64495)
-- OR
-- (autnum >= 131072 AND autnum <= 4199999999)
-- );
--
-- -- Delete everything that we have not seen for 14 days
-- DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
-- """)
--
-- def _handle_update_announcements_from_bird(self, server):
-- # Pre-compile the regular expression for faster searching
-- route = re.compile(b"^\s(.+?)\s+.+?\[(?:AS(.*?))?.\]$")
--
-- log.info("Requesting routing table from Bird (%s)" % server)
--
-- aggregated_networks = []
--
-- # Send command to list all routes
-- for line in self._bird_cmd(server, "show route"):
-- m = route.match(line)
-- if not m:
-- # Skip empty lines
-- if not line:
-- pass
--
-- # Ignore any header lines with the name of the routing table
-- elif line.startswith(b"Table"):
-- pass
--
-- # Log anything else
-- else:
-- log.debug("Could not parse line: %s" % line.decode())
--
-- continue
--
-- # Fetch the extracted network and ASN
-- network, autnum = m.groups()
--
-- # Decode into strings
-- if network:
-- network = network.decode()
-- if autnum:
-- autnum = autnum.decode()
--
-- # Collect all aggregated networks
-- if not autnum:
-- log.debug("%s is an aggregated network" % network)
-- aggregated_networks.append(network)
-- continue
--
-- # Insert it into the database
-- self.db.execute("INSERT INTO announcements(network, autnum) \
-- VALUES(%s, %s) ON CONFLICT (network) DO \
-- UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
-- network, autnum,
-- )
--
-- # Process any aggregated networks
-- for network in aggregated_networks:
-- log.debug("Processing aggregated network %s" % network)
--
-- # Run "show route all" for each network
-- for line in self._bird_cmd(server, "show route %s all" % network):
-- # Try finding the path
-- m = re.match(b"\s+BGP\.as_path:.* (\d+) {\d+}$", line)
-- if m:
-- # Select the last AS number in the path
-- autnum = m.group(1).decode()
--
-- # Insert it into the database
-- self.db.execute("INSERT INTO announcements(network, autnum) \
-- VALUES(%s, %s) ON CONFLICT (network) DO \
-- UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
-- network, autnum,
-- )
--
-- # We don't need to process any more
-- break
--
-- def _handle_update_announcements_from_telnet(self, server):
-- # Pre-compile regular expression for routes
-- route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
--
-- with telnetlib.Telnet(server) as t:
-- # Enable debug mode
-- #if ns.debug:
-- # t.set_debuglevel(10)
--
-- # Wait for console greeting
-- greeting = t.read_until(b"> ", timeout=30)
-- if not greeting:
-- log.error("Could not get a console prompt")
-- return 1
--
-- # Disable pagination
-- t.write(b"terminal length 0\n")
--
-- # Wait for the prompt to return
-- t.read_until(b"> ")
--
-- # Fetch the routing tables
-- for protocol in ("ipv6", "ipv4"):
-- log.info("Requesting %s routing table" % protocol)
--
-- # Request the full unicast routing table
-- t.write(b"show bgp %s unicast\n" % protocol.encode())
--
-- # Read entire header which ends with "Path"
-- t.read_until(b"Path\r\n")
--
-- while True:
-- # Try reading a full entry
-- # Those might be broken across multiple lines but ends with i
-- line = t.read_until(b"i\r\n", timeout=5)
-- if not line:
-- break
--
-- # Show line for debugging
-- #log.debug(repr(line))
--
-- # Try finding a route in here
-- m = route.match(line)
-- if m:
-- network, autnum = m.groups()
--
-- # Convert network to string
-- network = network.decode()
--
-- # Append /24 for IPv4 addresses
-- if not "/" in network and not ":" in network:
-- network = "%s/24" % network
--
-- # Convert AS number to integer
-- autnum = int(autnum)
--
-- log.info("Found announcement for %s by %s" % (network, autnum))
--
-- self.db.execute("INSERT INTO announcements(network, autnum) \
-- VALUES(%s, %s) ON CONFLICT (network) DO \
-- UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
-- network, autnum,
-- )
--
-- log.info("Finished reading the %s routing table" % protocol)
--
-- def _bird_cmd(self, socket_path, command):
-- # Connect to the socket
-- s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-- s.connect(socket_path)
--
-- # Allocate some buffer
-- buffer = b""
--
-- log.debug("Sending Bird command: %s" % command)
--
-- # Send the command
-- s.send(b"%s\n" % command.encode())
--
-- while True:
-- # Fill up the buffer
-- buffer += s.recv(4096)
--
-- while True:
-- # Search for the next newline
-- pos = buffer.find(b"\n")
--
-- # If we cannot find one, we go back and read more data
-- if pos <= 0:
-- break
--
-- # Cut after the newline character
-- pos += 1
--
-- # Split the line we want and keep the rest in buffer
-- line, buffer = buffer[:pos], buffer[pos:]
--
-- # Try parsing any status lines
-- if len(line) > 4 and line[:4].isdigit() and line[4] in (32, 45):
-- code, delim, line = int(line[:4]), line[4], line[5:]
--
-- log.debug("Received response code %s from bird" % code)
--
-- # End of output
-- if code == 0:
-- return
--
-- # Ignore hello line
-- elif code == 1:
-- continue
--
-- # Otherwise return the line
-- yield line
--
-- def handle_update_overrides(self, ns):
-- with self.db.transaction():
-- # Drop all data that we have
-- self.db.execute("""
-- TRUNCATE TABLE autnum_overrides;
-- TRUNCATE TABLE network_overrides;
-- """)
--
-- # Update overrides for various cloud providers big enough to publish their own IP
-- # network allocation lists in a machine-readable format...
-- self._update_overrides_for_aws()
--
-- # Update overrides for Spamhaus DROP feeds...
-- self._update_overrides_for_spamhaus_drop()
--
-- for file in ns.files:
-- log.info("Reading %s..." % file)
--
-- with open(file, "rb") as f:
-- for type, block in location.importer.read_blocks(f):
-- if type == "net":
-- network = block.get("net")
-- # Try to parse and normalise the network
-- try:
-- network = ipaddress.ip_network(network, strict=False)
-- except ValueError as e:
-- log.warning("Invalid IP network: %s: %s" % (network, e))
-- continue
--
-- # Prevent that we overwrite all networks
-- if network.prefixlen == 0:
-- log.warning("Skipping %s: You cannot overwrite default" % network)
-- continue
--
-- self.db.execute("""
-- INSERT INTO network_overrides(
-- network,
-- country,
-- source,
-- is_anonymous_proxy,
-- is_satellite_provider,
-- is_anycast,
-- is_drop
-- ) VALUES (%s, %s, %s, %s, %s, %s, %s)
-- ON CONFLICT (network) DO NOTHING""",
-- "%s" % network,
-- block.get("country"),
-- "manual",
-- self._parse_bool(block, "is-anonymous-proxy"),
-- self._parse_bool(block, "is-satellite-provider"),
-- self._parse_bool(block, "is-anycast"),
-- self._parse_bool(block, "drop"),
-- )
--
-- elif type == "aut-num":
-- autnum = block.get("aut-num")
--
-- # Check if AS number begins with "AS"
-- if not autnum.startswith("AS"):
-- log.warning("Invalid AS number: %s" % autnum)
-- continue
--
-- # Strip "AS"
-- autnum = autnum[2:]
--
-- self.db.execute("""
-- INSERT INTO autnum_overrides(
-- number,
-- name,
-- country,
-- source,
-- is_anonymous_proxy,
-- is_satellite_provider,
-- is_anycast,
-- is_drop
-- ) VALUES(%s, %s, %s, %s, %s, %s, %s, %s)
-- ON CONFLICT DO NOTHING""",
-- autnum,
-- block.get("name"),
-- block.get("country"),
-- "manual",
-- self._parse_bool(block, "is-anonymous-proxy"),
-- self._parse_bool(block, "is-satellite-provider"),
-- self._parse_bool(block, "is-anycast"),
-- self._parse_bool(block, "drop"),
-- )
--
-- else:
-- log.warning("Unsupported type: %s" % type)
--
-- def _update_overrides_for_aws(self):
-- # Download Amazon AWS IP allocation file to create overrides...
-- downloader = location.importer.Downloader()
--
-- try:
-- with downloader.request("https://ip-ranges.amazonaws.com/ip-ranges.json", return_blocks=False) as f:
-- aws_ip_dump = json.load(f.body)
-- except Exception as e:
-- log.error("unable to preprocess Amazon AWS IP ranges: %s" % e)
-- return
--
-- # XXX: Set up a dictionary for mapping a region name to a country. Unfortunately,
-- # there seems to be no machine-readable version available of this other than
-- # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html
-- # (worse, it seems to be incomplete :-/ ); https://www.cloudping.cloud/endpoints
-- # was helpful here as well.
-- aws_region_country_map = {
-- "af-south-1": "ZA",
-- "ap-east-1": "HK",
-- "ap-south-1": "IN",
-- "ap-south-2": "IN",
-- "ap-northeast-3": "JP",
-- "ap-northeast-2": "KR",
-- "ap-southeast-1": "SG",
-- "ap-southeast-2": "AU",
-- "ap-southeast-3": "MY",
-- "ap-southeast-4": "AU",
-- "ap-northeast-1": "JP",
-- "ca-central-1": "CA",
-- "eu-central-1": "DE",
-- "eu-central-2": "CH",
-- "eu-west-1": "IE",
-- "eu-west-2": "GB",
-- "eu-south-1": "IT",
-- "eu-south-2": "ES",
-- "eu-west-3": "FR",
-- "eu-north-1": "SE",
-- "il-central-1": "IL", # XXX: This one is not documented anywhere except for ip-ranges.json itself
-- "me-central-1": "AE",
-- "me-south-1": "BH",
-- "sa-east-1": "BR"
-- }
--
-- # Fetch all valid country codes to check parsed networks aganist...
-- rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
-- validcountries = []
--
-- for row in rows:
-- validcountries.append(row.country_code)
--
-- with self.db.transaction():
-- for snetwork in aws_ip_dump["prefixes"] + aws_ip_dump["ipv6_prefixes"]:
-- try:
-- network = ipaddress.ip_network(snetwork.get("ip_prefix") or snetwork.get("ipv6_prefix"), strict=False)
-- except ValueError:
-- log.warning("Unable to parse line: %s" % snetwork)
-- continue
--
-- # Sanitize parsed networks...
-- if not self._check_parsed_network(network):
-- continue
--
-- # Determine region of this network...
-- region = snetwork["region"]
-- cc = None
-- is_anycast = False
--
-- # Any region name starting with "us-" will get "US" country code assigned straight away...
-- if region.startswith("us-"):
-- cc = "US"
-- elif region.startswith("cn-"):
-- # ... same goes for China ...
-- cc = "CN"
-- elif region == "GLOBAL":
-- # ... funny region name for anycast-like networks ...
-- is_anycast = True
-- elif region in aws_region_country_map:
-- # ... assign looked up country code otherwise ...
-- cc = aws_region_country_map[region]
-- else:
-- # ... and bail out if we are missing something here
-- log.warning("Unable to determine country code for line: %s" % snetwork)
-- continue
--
-- # Skip networks with unknown country codes
-- if not is_anycast and validcountries and cc not in validcountries:
-- log.warning("Skipping Amazon AWS network with bogus country '%s': %s" % \
-- (cc, network))
-- return
--
-- # Conduct SQL statement...
-- self.db.execute("""
-- INSERT INTO network_overrides(
-- network,
-- country,
-- source,
-- is_anonymous_proxy,
-- is_satellite_provider,
-- is_anycast
-- ) VALUES (%s, %s, %s, %s, %s, %s)
-- ON CONFLICT (network) DO NOTHING""",
-- "%s" % network,
-- cc,
-- "Amazon AWS IP feed",
-- None,
-- None,
-- is_anycast,
-- )
--
--
-- def _update_overrides_for_spamhaus_drop(self):
-- downloader = location.importer.Downloader()
--
-- ip_urls = [
-- "https://www.spamhaus.org/drop/drop.txt",
-- "https://www.spamhaus.org/drop/edrop.txt",
-- "https://www.spamhaus.org/drop/dropv6.txt"
-- ]
--
-- asn_urls = [
-- "https://www.spamhaus.org/drop/asndrop.txt"
-- ]
--
-- for url in ip_urls:
-- try:
-- with downloader.request(url, return_blocks=False) as f:
-- fcontent = f.body.readlines()
-- except Exception as e:
-- log.error("Unable to download Spamhaus DROP URL %s: %s" % (url, e))
-- return
--
-- # Iterate through every line, filter comments and add remaining networks to
-- # the override table in case they are valid...
-- with self.db.transaction():
-- for sline in fcontent:
--
-- # The response is assumed to be encoded in UTF-8...
-- sline = sline.decode("utf-8")
--
-- # Comments start with a semicolon...
-- if sline.startswith(";"):
-- continue
--
-- # Extract network and ignore anything afterwards...
-- try:
-- network = ipaddress.ip_network(sline.split()[0], strict=False)
-- except ValueError:
-- log.error("Unable to parse line: %s" % sline)
-- continue
--
-- # Sanitize parsed networks...
-- if not self._check_parsed_network(network):
-- log.warning("Skipping bogus network found in Spamhaus DROP URL %s: %s" % \
-- (url, network))
-- continue
--
-- # Conduct SQL statement...
-- self.db.execute("""
-- INSERT INTO network_overrides(
-- network,
-- source,
-- is_drop
-- ) VALUES (%s, %s, %s)
-- ON CONFLICT (network) DO UPDATE SET is_drop = True""",
-- "%s" % network,
-- "Spamhaus DROP lists",
-- True
-- )
--
-- for url in asn_urls:
-- try:
-- with downloader.request(url, return_blocks=False) as f:
-- fcontent = f.body.readlines()
-- except Exception as e:
-- log.error("Unable to download Spamhaus DROP URL %s: %s" % (url, e))
-- return
--
-- # Iterate through every line, filter comments and add remaining ASNs to
-- # the override table in case they are valid...
-- with self.db.transaction():
-- for sline in fcontent:
--
-- # The response is assumed to be encoded in UTF-8...
-- sline = sline.decode("utf-8")
--
-- # Comments start with a semicolon...
-- if sline.startswith(";"):
-- continue
--
-- # Throw away anything after the first space...
-- sline = sline.split()[0]
--
-- # ... strip the "AS" prefix from it ...
-- sline = sline.strip("AS")
--
-- # ... and convert it into an integer. Voila.
-- asn = int(sline)
--
-- # Filter invalid ASNs...
-- if not self._check_parsed_asn(asn):
-- log.warning("Skipping bogus ASN found in Spamhaus DROP URL %s: %s" % \
-- (url, asn))
-- continue
--
-- # Conduct SQL statement...
-- self.db.execute("""
-- INSERT INTO autnum_overrides(
-- number,
-- source,
-- is_drop
-- ) VALUES (%s, %s, %s)
-- ON CONFLICT (number) DO UPDATE SET is_drop = True""",
-- "%s" % asn,
-- "Spamhaus ASN-DROP list",
-- True
-- )
--
-- @staticmethod
-- def _parse_bool(block, key):
-- val = block.get(key)
--
-- # There is no point to proceed when we got None
-- if val is None:
-- return
--
-- # Convert to lowercase
-- val = val.lower()
--
-- # True
-- if val in ("yes", "1"):
-- return True
--
-- # False
-- if val in ("no", "0"):
-- return False
--
-- # Default to None
-- return None
--
-- def handle_import_countries(self, ns):
-- with self.db.transaction():
-- # Drop all data that we have
-- self.db.execute("TRUNCATE TABLE countries")
--
-- for file in ns.file:
-- for line in file:
-- line = line.rstrip()
--
-- # Ignore any comments
-- if line.startswith("#"):
-- continue
--
-- try:
-- country_code, continent_code, name = line.split(maxsplit=2)
-- except:
-- log.warning("Could not parse line: %s" % line)
-- continue
--
-- self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
-- VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
--
--
--def split_line(line):
-- key, colon, val = line.partition(":")
--
-- # Strip any excess space
-- key = key.strip()
-- val = val.strip()
--
-- return key, val
--
--def main():
-- # Run the command line interface
-- c = CLI()
-- c.run()
--
--main()
-diff --git a/src/python/location.in b/src/python/location.in
-deleted file mode 100644
-index 233cea0..0000000
---- a/src/python/location.in
-+++ /dev/null
-@@ -1,644 +0,0 @@
--#!/usr/bin/python3
--###############################################################################
--# #
--# libloc - A library to determine the location of someone on the Internet #
--# #
--# Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org> #
--# #
--# This library is free software; you can redistribute it and/or #
--# modify it under the terms of the GNU Lesser General Public #
--# License as published by the Free Software Foundation; either #
--# version 2.1 of the License, or (at your option) any later version. #
--# #
--# This library is distributed in the hope that it will be useful, #
--# but WITHOUT ANY WARRANTY; without even the implied warranty of #
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
--# Lesser General Public License for more details. #
--# #
--###############################################################################
--
--import argparse
--import datetime
--import ipaddress
--import logging
--import os
--import re
--import shutil
--import socket
--import sys
--import time
--
--# Load our location module
--import location
--import location.downloader
--import location.export
--
--from location.i18n import _
--
--# Setup logging
--log = logging.getLogger("location")
--
--# Output formatters
--
--class CLI(object):
-- def parse_cli(self):
-- parser = argparse.ArgumentParser(
-- description=_("Location Database Command Line Interface"),
-- )
-- subparsers = parser.add_subparsers()
--
-- # Global configuration flags
-- parser.add_argument("--debug", action="store_true",
-- help=_("Enable debug output"))
-- parser.add_argument("--quiet", action="store_true",
-- help=_("Enable quiet mode"))
--
-- # version
-- parser.add_argument("--version", action="version",
-- version="%(prog)s @VERSION@")
--
-- # database
-- parser.add_argument("--database", "-d",
-- default="@databasedir@/database.db", help=_("Path to database"),
-- )
--
-- # public key
-- parser.add_argument("--public-key", "-k",
-- default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
-- )
--
-- # Show the database version
-- version = subparsers.add_parser("version",
-- help=_("Show database version"))
-- version.set_defaults(func=self.handle_version)
--
-- # lookup an IP address
-- lookup = subparsers.add_parser("lookup",
-- help=_("Lookup one or multiple IP addresses"),
-- )
-- lookup.add_argument("address", nargs="+")
-- lookup.set_defaults(func=self.handle_lookup)
--
-- # Dump the whole database
-- dump = subparsers.add_parser("dump",
-- help=_("Dump the entire database"),
-- )
-- dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
-- dump.set_defaults(func=self.handle_dump)
--
-- # Update
-- update = subparsers.add_parser("update", help=_("Update database"))
-- update.add_argument("--cron",
-- help=_("Update the library only once per interval"),
-- choices=("daily", "weekly", "monthly"),
-- )
-- update.set_defaults(func=self.handle_update)
--
-- # Verify
-- verify = subparsers.add_parser("verify",
-- help=_("Verify the downloaded database"))
-- verify.set_defaults(func=self.handle_verify)
--
-- # Get AS
-- get_as = subparsers.add_parser("get-as",
-- help=_("Get information about one or multiple Autonomous Systems"),
-- )
-- get_as.add_argument("asn", nargs="+")
-- get_as.set_defaults(func=self.handle_get_as)
--
-- # Search for AS
-- search_as = subparsers.add_parser("search-as",
-- help=_("Search for Autonomous Systems that match the string"),
-- )
-- search_as.add_argument("query", nargs=1)
-- search_as.set_defaults(func=self.handle_search_as)
--
-- # List all networks in an AS
-- list_networks_by_as = subparsers.add_parser("list-networks-by-as",
-- help=_("Lists all networks in an AS"),
-- )
-- list_networks_by_as.add_argument("asn", nargs=1, type=int)
-- list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
-- list_networks_by_as.add_argument("--format",
-- choices=location.export.formats.keys(), default="list")
-- list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
--
-- # List all networks in a country
-- list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
-- help=_("Lists all networks in a country"),
-- )
-- list_networks_by_cc.add_argument("country_code", nargs=1)
-- list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
-- list_networks_by_cc.add_argument("--format",
-- choices=location.export.formats.keys(), default="list")
-- list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
--
-- # List all networks with flags
-- list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
-- help=_("Lists all networks with flags"),
-- )
-- list_networks_by_flags.add_argument("--anonymous-proxy",
-- action="store_true", help=_("Anonymous Proxies"),
-- )
-- list_networks_by_flags.add_argument("--satellite-provider",
-- action="store_true", help=_("Satellite Providers"),
-- )
-- list_networks_by_flags.add_argument("--anycast",
-- action="store_true", help=_("Anycasts"),
-- )
-- list_networks_by_flags.add_argument("--drop",
-- action="store_true", help=_("Hostile Networks safe to drop"),
-- )
-- list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
-- list_networks_by_flags.add_argument("--format",
-- choices=location.export.formats.keys(), default="list")
-- list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
--
-- # List bogons
-- list_bogons = subparsers.add_parser("list-bogons",
-- help=_("Lists all bogons"),
-- )
-- list_bogons.add_argument("--family", choices=("ipv6", "ipv4"))
-- list_bogons.add_argument("--format",
-- choices=location.export.formats.keys(), default="list")
-- list_bogons.set_defaults(func=self.handle_list_bogons)
--
-- # List countries
-- list_countries = subparsers.add_parser("list-countries",
-- help=_("Lists all countries"),
-- )
-- list_countries.add_argument("--show-name",
-- action="store_true", help=_("Show the name of the country"),
-- )
-- list_countries.add_argument("--show-continent",
-- action="store_true", help=_("Show the continent"),
-- )
-- list_countries.set_defaults(func=self.handle_list_countries)
--
-- # Export
-- export = subparsers.add_parser("export",
-- help=_("Exports data in many formats to load it into packet filters"),
-- )
-- export.add_argument("--format", help=_("Output format"),
-- choices=location.export.formats.keys(), default="list")
-- export.add_argument("--directory", help=_("Output directory"))
-- export.add_argument("--family",
-- help=_("Specify address family"), choices=("ipv6", "ipv4"),
-- )
-- export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
-- export.set_defaults(func=self.handle_export)
--
-- args = parser.parse_args()
--
-- # Configure logging
-- if args.debug:
-- location.logger.set_level(logging.DEBUG)
-- elif args.quiet:
-- location.logger.set_level(logging.WARNING)
--
-- # Print usage if no action was given
-- if not "func" in args:
-- parser.print_usage()
-- sys.exit(2)
--
-- return args
--
-- def run(self):
-- # Parse command line arguments
-- args = self.parse_cli()
--
-- # Open database
-- try:
-- db = location.Database(args.database)
-- except FileNotFoundError as e:
-- # Allow continuing without a database
-- if args.func == self.handle_update:
-- db = None
--
-- else:
-- sys.stderr.write("location: Could not open database %s: %s\n" \
-- % (args.database, e))
-- sys.exit(1)
--
-- # Translate family (if present)
-- if "family" in args:
-- if args.family == "ipv6":
-- args.family = socket.AF_INET6
-- elif args.family == "ipv4":
-- args.family = socket.AF_INET
-- else:
-- args.family = 0
--
-- # Call function
-- try:
-- ret = args.func(db, args)
--
-- # Catch invalid inputs
-- except ValueError as e:
-- sys.stderr.write("%s\n" % e)
-- ret = 2
--
-- # Catch any other exceptions
-- except Exception as e:
-- sys.stderr.write("%s\n" % e)
-- ret = 1
--
-- # Return with exit code
-- if ret:
-- sys.exit(ret)
--
-- # Otherwise just exit
-- sys.exit(0)
--
-- def handle_version(self, db, ns):
-- """
-- Print the version of the database
-- """
-- t = time.strftime(
-- "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
-- )
--
-- print(t)
--
-- def handle_lookup(self, db, ns):
-- ret = 0
--
-- format = " %-24s: %s"
--
-- for address in ns.address:
-- try:
-- network = db.lookup(address)
-- except ValueError:
-- print(_("Invalid IP address: %s") % address, file=sys.stderr)
-- return 2
--
-- args = {
-- "address" : address,
-- "network" : network,
-- }
--
-- # Nothing found?
-- if not network:
-- print(_("Nothing found for %(address)s") % args, file=sys.stderr)
-- ret = 1
-- continue
--
-- print("%s:" % address)
-- print(format % (_("Network"), network))
--
-- # Print country
-- if network.country_code:
-- country = db.get_country(network.country_code)
--
-- print(format % (
-- _("Country"),
-- country.name if country else network.country_code),
-- )
--
-- # Print AS information
-- if network.asn:
-- autonomous_system = db.get_as(network.asn)
--
-- print(format % (
-- _("Autonomous System"),
-- autonomous_system or "AS%s" % network.asn),
-- )
--
-- # Anonymous Proxy
-- if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
-- print(format % (
-- _("Anonymous Proxy"), _("yes"),
-- ))
--
-- # Satellite Provider
-- if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
-- print(format % (
-- _("Satellite Provider"), _("yes"),
-- ))
--
-- # Anycast
-- if network.has_flag(location.NETWORK_FLAG_ANYCAST):
-- print(format % (
-- _("Anycast"), _("yes"),
-- ))
--
-- # Hostile Network
-- if network.has_flag(location.NETWORK_FLAG_DROP):
-- print(format % (
-- _("Hostile Network safe to drop"), _("yes"),
-- ))
--
-- return ret
--
-- def handle_dump(self, db, ns):
-- # Use output file or write to stdout
-- f = ns.output or sys.stdout
--
-- # Format everything like this
-- format = "%-24s %s\n"
--
-- # Write metadata
-- f.write("#\n# Location Database Export\n#\n")
--
-- f.write("# Generated: %s\n" % time.strftime(
-- "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
-- ))
--
-- if db.vendor:
-- f.write("# Vendor: %s\n" % db.vendor)
--
-- if db.license:
-- f.write("# License: %s\n" % db.license)
--
-- f.write("#\n")
--
-- if db.description:
-- for line in db.description.splitlines():
-- line = "# %s" % line
-- f.write("%s\n" % line.rstrip())
--
-- f.write("#\n")
--
-- # Iterate over all ASes
-- for a in db.ases:
-- f.write("\n")
-- f.write(format % ("aut-num:", "AS%s" % a.number))
-- f.write(format % ("name:", a.name))
--
-- flags = {
-- location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:",
-- location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
-- location.NETWORK_FLAG_ANYCAST : "is-anycast:",
-- location.NETWORK_FLAG_DROP : "drop:",
-- }
--
-- # Iterate over all networks
-- for n in db.networks:
-- f.write("\n")
-- f.write(format % ("net:", n))
--
-- if n.country_code:
-- f.write(format % ("country:", n.country_code))
--
-- if n.asn:
-- f.write(format % ("aut-num:", n.asn))
--
-- # Print all flags
-- for flag in flags:
-- if n.has_flag(flag):
-- f.write(format % (flags[flag], "yes"))
--
-- def handle_get_as(self, db, ns):
-- """
-- Gets information about Autonomous Systems
-- """
-- ret = 0
--
-- for asn in ns.asn:
-- try:
-- asn = int(asn)
-- except ValueError:
-- print(_("Invalid ASN: %s") % asn, file=sys.stderr)
-- ret = 1
-- continue
--
-- # Fetch AS from database
-- a = db.get_as(asn)
--
-- # Nothing found
-- if not a:
-- print(_("Could not find AS%s") % asn, file=sys.stderr)
-- ret = 1
-- continue
--
-- print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
--
-- return ret
--
-- def handle_search_as(self, db, ns):
-- for query in ns.query:
-- # Print all matches ASes
-- for a in db.search_as(query):
-- print(a)
--
-- def handle_update(self, db, ns):
-- if ns.cron and db:
-- now = time.time()
--
-- if ns.cron == "daily":
-- delta = datetime.timedelta(days=1)
-- elif ns.cron == "weekly":
-- delta = datetime.timedelta(days=7)
-- elif ns.cron == "monthly":
-- delta = datetime.timedelta(days=30)
--
-- delta = delta.total_seconds()
--
-- # Check if the database has recently been updated
-- if db.created_at >= (now - delta):
-- log.info(
-- _("The database has been updated recently"),
-- )
-- return 3
--
-- # Fetch the timestamp we need from DNS
-- t = location.discover_latest_version()
--
-- # Check the version of the local database
-- if db and t and db.created_at >= t:
-- log.info("Already on the latest version")
-- return
--
-- # Download the database into the correct directory
-- tmpdir = os.path.dirname(ns.database)
--
-- # Create a downloader
-- d = location.downloader.Downloader()
--
-- # Try downloading a new database
-- try:
-- t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
--
-- # If no file could be downloaded, log a message
-- except FileNotFoundError as e:
-- log.error("Could not download a new database")
-- return 1
--
-- # If we have not received a new file, there is nothing to do
-- if not t:
-- return 3
--
-- # Move temporary file to destination
-- shutil.move(t.name, ns.database)
--
-- return 0
--
-- def handle_verify(self, db, ns):
-- # Verify the database
-- with open(ns.public_key, "r") as f:
-- if not db.verify(f):
-- log.error("Could not verify database")
-- return 1
--
-- # Success
-- log.debug("Database successfully verified")
-- return 0
--
-- def __get_output_formatter(self, ns):
-- try:
-- cls = location.export.formats[ns.format]
-- except KeyError:
-- cls = location.export.OutputFormatter
--
-- return cls
--
-- def handle_list_countries(self, db, ns):
-- for country in db.countries:
-- line = [
-- country.code,
-- ]
--
-- if ns.show_continent:
-- line.append(country.continent_code)
--
-- if ns.show_name:
-- line.append(country.name)
--
-- # Format the output
-- line = " ".join(line)
--
-- # Print the output
-- print(line)
--
-- def handle_list_networks_by_as(self, db, ns):
-- writer = self.__get_output_formatter(ns)
--
-- for asn in ns.asn:
-- f = writer("AS%s" % asn, f=sys.stdout)
--
-- # Print all matching networks
-- for n in db.search_networks(asns=[asn], family=ns.family):
-- f.write(n)
--
-- f.finish()
--
-- def handle_list_networks_by_cc(self, db, ns):
-- writer = self.__get_output_formatter(ns)
--
-- for country_code in ns.country_code:
-- # Open standard output
-- f = writer(country_code, f=sys.stdout)
--
-- # Print all matching networks
-- for n in db.search_networks(country_codes=[country_code], family=ns.family):
-- f.write(n)
--
-- f.finish()
--
-- def handle_list_networks_by_flags(self, db, ns):
-- flags = 0
--
-- if ns.anonymous_proxy:
-- flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
--
-- if ns.satellite_provider:
-- flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
--
-- if ns.anycast:
-- flags |= location.NETWORK_FLAG_ANYCAST
--
-- if ns.drop:
-- flags |= location.NETWORK_FLAG_DROP
--
-- if not flags:
-- raise ValueError(_("You must at least pass one flag"))
--
-- writer = self.__get_output_formatter(ns)
-- f = writer("custom", f=sys.stdout)
--
-- for n in db.search_networks(flags=flags, family=ns.family):
-- f.write(n)
--
-- f.finish()
--
-- def handle_list_bogons(self, db, ns):
-- writer = self.__get_output_formatter(ns)
-- f = writer("bogons", f=sys.stdout)
--
-- for n in db.list_bogons(family=ns.family):
-- f.write(n)
--
-- f.finish()
--
-- def handle_export(self, db, ns):
-- countries, asns = [], []
--
-- # Translate family
-- if ns.family:
-- families = [ ns.family ]
-- else:
-- families = [ socket.AF_INET6, socket.AF_INET ]
--
-- for object in ns.objects:
-- m = re.match("^AS(\d+)$", object)
-- if m:
-- object = int(m.group(1))
--
-- asns.append(object)
--
-- elif location.country_code_is_valid(object) \
-- or object in ("A1", "A2", "A3", "XD"):
-- countries.append(object)
--
-- else:
-- log.warning("Invalid argument: %s" % object)
-- continue
--
-- # Default to exporting all countries
-- if not countries and not asns:
-- countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries]
--
-- # Select the output format
-- writer = self.__get_output_formatter(ns)
--
-- e = location.export.Exporter(db, writer)
-- e.export(ns.directory, countries=countries, asns=asns, families=families)
--
--
--def format_timedelta(t):
-- s = []
--
-- if t.days:
-- s.append(
-- _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
-- )
--
-- hours = t.seconds // 3600
-- if hours:
-- s.append(
-- _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
-- )
--
-- minutes = (t.seconds % 3600) // 60
-- if minutes:
-- s.append(
-- _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
-- )
--
-- seconds = t.seconds % 60
-- if t.seconds:
-- s.append(
-- _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
-- )
--
-- if not s:
-- return _("Now")
--
-- return _("%s ago") % ", ".join(s)
--
--def main():
-- # Run the command line interface
-- c = CLI()
-- c.run()
--
--main()
-diff --git a/src/python/location/__init__.py b/src/python/location/__init__.py
-new file mode 100644
-index 0000000..9b570c7
---- /dev/null
-+++ b/src/python/location/__init__.py
-@@ -0,0 +1,24 @@
-+#!/usr/bin/python3
-+###############################################################################
-+# #
-+# libloc - A library to determine the location of someone on the Internet #
-+# #
-+# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
-+# #
-+# This library is free software; you can redistribute it and/or #
-+# modify it under the terms of the GNU Lesser General Public #
-+# License as published by the Free Software Foundation; either #
-+# version 2.1 of the License, or (at your option) any later version. #
-+# #
-+# This library is distributed in the hope that it will be useful, #
-+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
-+# Lesser General Public License for more details. #
-+# #
-+###############################################################################
-+
-+# Import everything from the C module
-+from _location import *
-+
-+# Initialise logging
-+from . import logger
-diff --git a/src/python/location/database.py b/src/python/location/database.py
-new file mode 100644
-index 0000000..5d79941
---- /dev/null
-+++ b/src/python/location/database.py
-@@ -0,0 +1,213 @@
-+#!/usr/bin/env python
-+
-+"""
-+ A lightweight wrapper around psycopg2.
-+
-+ Originally part of the Tornado framework. The tornado.database module
-+ is slated for removal in Tornado 3.0, and it is now available separately
-+ as torndb.
-+"""
-+
-+import logging
-+import psycopg2
-+
-+log = logging.getLogger("location.database")
-+log.propagate = 1
-+
-+class Connection(object):
-+ """
-+ A lightweight wrapper around MySQLdb DB-API connections.
-+
-+ The main value we provide is wrapping rows in a dict/object so that
-+ columns can be accessed by name. Typical usage::
-+
-+ db = torndb.Connection("localhost", "mydatabase")
-+ for article in db.query("SELECT * FROM articles"):
-+ print article.title
-+
-+ Cursors are hidden by the implementation, but other than that, the methods
-+ are very similar to the DB-API.
-+
-+ We explicitly set the timezone to UTC and the character encoding to
-+ UTF-8 on all connections to avoid time zone and encoding errors.
-+ """
-+ def __init__(self, host, database, user=None, password=None):
-+ self.host = host
-+ self.database = database
-+
-+ self._db = None
-+ self._db_args = {
-+ "host" : host,
-+ "database" : database,
-+ "user" : user,
-+ "password" : password,
-+ "sslmode" : "require",
-+ }
-+
-+ try:
-+ self.reconnect()
-+ except Exception:
-+ log.error("Cannot connect to database on %s", self.host, exc_info=True)
-+
-+ def __del__(self):
-+ self.close()
-+
-+ def close(self):
-+ """
-+ Closes this database connection.
-+ """
-+ if getattr(self, "_db", None) is not None:
-+ self._db.close()
-+ self._db = None
-+
-+ def reconnect(self):
-+ """
-+ Closes the existing database connection and re-opens it.
-+ """
-+ self.close()
-+
-+ self._db = psycopg2.connect(**self._db_args)
-+ self._db.autocommit = True
-+
-+ # Initialize the timezone setting.
-+ self.execute("SET TIMEZONE TO 'UTC'")
-+
-+ def query(self, query, *parameters, **kwparameters):
-+ """
-+ Returns a row list for the given query and parameters.
-+ """
-+ cursor = self._cursor()
-+ try:
-+ self._execute(cursor, query, parameters, kwparameters)
-+ column_names = [d[0] for d in cursor.description]
-+ return [Row(zip(column_names, row)) for row in cursor]
-+ finally:
-+ cursor.close()
-+
-+ def get(self, query, *parameters, **kwparameters):
-+ """
-+ Returns the first row returned for the given query.
-+ """
-+ rows = self.query(query, *parameters, **kwparameters)
-+ if not rows:
-+ return None
-+ elif len(rows) > 1:
-+ raise Exception("Multiple rows returned for Database.get() query")
-+ else:
-+ return rows[0]
-+
-+ def execute(self, query, *parameters, **kwparameters):
-+ """
-+ Executes the given query, returning the lastrowid from the query.
-+ """
-+ return self.execute_lastrowid(query, *parameters, **kwparameters)
-+
-+ def execute_lastrowid(self, query, *parameters, **kwparameters):
-+ """
-+ Executes the given query, returning the lastrowid from the query.
-+ """
-+ cursor = self._cursor()
-+ try:
-+ self._execute(cursor, query, parameters, kwparameters)
-+ return cursor.lastrowid
-+ finally:
-+ cursor.close()
-+
-+ def execute_rowcount(self, query, *parameters, **kwparameters):
-+ """
-+ Executes the given query, returning the rowcount from the query.
-+ """
-+ cursor = self._cursor()
-+ try:
-+ self._execute(cursor, query, parameters, kwparameters)
-+ return cursor.rowcount
-+ finally:
-+ cursor.close()
-+
-+ def executemany(self, query, parameters):
-+ """
-+ Executes the given query against all the given param sequences.
-+
-+ We return the lastrowid from the query.
-+ """
-+ return self.executemany_lastrowid(query, parameters)
-+
-+ def executemany_lastrowid(self, query, parameters):
-+ """
-+ Executes the given query against all the given param sequences.
-+
-+ We return the lastrowid from the query.
-+ """
-+ cursor = self._cursor()
-+ try:
-+ cursor.executemany(query, parameters)
-+ return cursor.lastrowid
-+ finally:
-+ cursor.close()
-+
-+ def executemany_rowcount(self, query, parameters):
-+ """
-+ Executes the given query against all the given param sequences.
-+
-+ We return the rowcount from the query.
-+ """
-+ cursor = self._cursor()
-+
-+ try:
-+ cursor.executemany(query, parameters)
-+ return cursor.rowcount
-+ finally:
-+ cursor.close()
-+
-+ def _ensure_connected(self):
-+ if self._db is None:
-+ log.warning("Database connection was lost...")
-+
-+ self.reconnect()
-+
-+ def _cursor(self):
-+ self._ensure_connected()
-+ return self._db.cursor()
-+
-+ def _execute(self, cursor, query, parameters, kwparameters):
-+ log.debug("SQL Query: %s" % (query % (kwparameters or parameters)))
-+
-+ try:
-+ return cursor.execute(query, kwparameters or parameters)
-+ except (OperationalError, psycopg2.ProgrammingError):
-+ log.error("Error connecting to database on %s", self.host)
-+ self.close()
-+ raise
-+
-+ def transaction(self):
-+ return Transaction(self)
-+
-+
-+class Row(dict):
-+ """A dict that allows for object-like property access syntax."""
-+ def __getattr__(self, name):
-+ try:
-+ return self[name]
-+ except KeyError:
-+ raise AttributeError(name)
-+
-+
-+class Transaction(object):
-+ def __init__(self, db):
-+ self.db = db
-+
-+ self.db.execute("START TRANSACTION")
-+
-+ def __enter__(self):
-+ return self
-+
-+ def __exit__(self, exctype, excvalue, traceback):
-+ if exctype is not None:
-+ self.db.execute("ROLLBACK")
-+ else:
-+ self.db.execute("COMMIT")
-+
-+
-+# Alias some common exceptions
-+IntegrityError = psycopg2.IntegrityError
-+OperationalError = psycopg2.OperationalError
-diff --git a/src/python/location/downloader.py b/src/python/location/downloader.py
-new file mode 100644
-index 0000000..05f7872
---- /dev/null
-+++ b/src/python/location/downloader.py
-@@ -0,0 +1,211 @@
-+#!/usr/bin/python3
-+###############################################################################
-+# #
-+# libloc - A library to determine the location of someone on the Internet #
-+# #
-+# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
-+# #
-+# This library is free software; you can redistribute it and/or #
-+# modify it under the terms of the GNU Lesser General Public #
-+# License as published by the Free Software Foundation; either #
-+# version 2.1 of the License, or (at your option) any later version. #
-+# #
-+# This library is distributed in the hope that it will be useful, #
-+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
-+# Lesser General Public License for more details. #
-+# #
-+###############################################################################
-+
-+import logging
-+import lzma
-+import os
-+import random
-+import stat
-+import tempfile
-+import time
-+import urllib.error
-+import urllib.parse
-+import urllib.request
-+
-+from . import __version__
-+from _location import Database, DATABASE_VERSION_LATEST
-+
-+DATABASE_FILENAME = "location.db.xz"
-+MIRRORS = (
-+ "https://location.ipfire.org/databases/",
-+)
-+
-+# Initialise logging
-+log = logging.getLogger("location.downloader")
-+log.propagate = 1
-+
-+class Downloader(object):
-+ def __init__(self, version=DATABASE_VERSION_LATEST, mirrors=None):
-+ self.version = version
-+
-+ # Set mirrors or use defaults
-+ self.mirrors = list(mirrors or MIRRORS)
-+
-+ # Randomize mirrors
-+ random.shuffle(self.mirrors)
-+
-+ # Get proxies from environment
-+ self.proxies = self._get_proxies()
-+
-+ def _get_proxies(self):
-+ proxies = {}
-+
-+ for protocol in ("https", "http"):
-+ proxy = os.environ.get("%s_proxy" % protocol, None)
-+
-+ if proxy:
-+ proxies[protocol] = proxy
-+
-+ return proxies
-+
-+ def _make_request(self, url, baseurl=None, headers={}):
-+ if baseurl:
-+ url = urllib.parse.urljoin(baseurl, url)
-+
-+ req = urllib.request.Request(url, method="GET")
-+
-+ # Update headers
-+ headers.update({
-+ "User-Agent" : "location/%s" % __version__,
-+ })
-+
-+ # Set headers
-+ for header in headers:
-+ req.add_header(header, headers[header])
-+
-+ # Set proxies
-+ for protocol in self.proxies:
-+ req.set_proxy(self.proxies[protocol], protocol)
-+
-+ return req
-+
-+ def _send_request(self, req, **kwargs):
-+ # Log request headers
-+ log.debug("HTTP %s Request to %s" % (req.method, req.host))
-+ log.debug(" URL: %s" % req.full_url)
-+ log.debug(" Headers:")
-+ for k, v in req.header_items():
-+ log.debug(" %s: %s" % (k, v))
-+
-+ try:
-+ res = urllib.request.urlopen(req, **kwargs)
-+
-+ except urllib.error.HTTPError as e:
-+ # Log response headers
-+ log.debug("HTTP Response: %s" % e.code)
-+ log.debug(" Headers:")
-+ for header in e.headers:
-+ log.debug(" %s: %s" % (header, e.headers[header]))
-+
-+ # Raise all other errors
-+ raise e
-+
-+ # Log response headers
-+ log.debug("HTTP Response: %s" % res.code)
-+ log.debug(" Headers:")
-+ for k, v in res.getheaders():
-+ log.debug(" %s: %s" % (k, v))
-+
-+ return res
-+
-+ def download(self, public_key, timestamp=None, tmpdir=None, **kwargs):
-+ url = "%s/%s" % (self.version, DATABASE_FILENAME)
-+
-+ headers = {}
-+ if timestamp:
-+ headers["If-Modified-Since"] = time.strftime(
-+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
-+ )
-+
-+ t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
-+ with t:
-+ # Try all mirrors
-+ for mirror in self.mirrors:
-+ # Prepare HTTP request
-+ req = self._make_request(url, baseurl=mirror, headers=headers)
-+
-+ try:
-+ with self._send_request(req) as res:
-+ decompressor = lzma.LZMADecompressor()
-+
-+ # Read all data
-+ while True:
-+ buf = res.read(1024)
-+ if not buf:
-+ break
-+
-+ # Decompress data
-+ buf = decompressor.decompress(buf)
-+ if buf:
-+ t.write(buf)
-+
-+ # Write all data to disk
-+ t.flush()
-+
-+ # Catch decompression errors
-+ except lzma.LZMAError as e:
-+ log.warning("Could not decompress downloaded file: %s" % e)
-+ continue
-+
-+ except urllib.error.HTTPError as e:
-+ # The file on the server was too old
-+ if e.code == 304:
-+ log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)
-+
-+ # Log any other HTTP errors
-+ else:
-+ log.warning("%s reported: %s" % (mirror, e))
-+
-+ # Throw away any downloaded content and try again
-+ t.truncate()
-+
-+ else:
-+ # Check if the downloaded database is recent
-+ if not self._check_database(t, public_key, timestamp):
-+ log.warning("Downloaded database is outdated. Trying next mirror...")
-+
-+ # Throw away the data and try again
-+ t.truncate()
-+ continue
-+
-+ # Make the file readable for everyone
-+ os.chmod(t.name, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)
-+
-+ # Return temporary file
-+ return t
-+
-+ # Delete the temporary file after unsuccessful downloads
-+ os.unlink(t.name)
-+
-+ raise FileNotFoundError(url)
-+
-+ def _check_database(self, f, public_key, timestamp=None):
-+ """
-+ Checks the downloaded database if it can be opened,
-+ verified and if it is recent enough
-+ """
-+ log.debug("Opening downloaded database at %s" % f.name)
-+
-+ db = Database(f.name)
-+
-+ # Database is not recent
-+ if timestamp and db.created_at < timestamp:
-+ return False
-+
-+ log.info("Downloaded new database from %s" % (time.strftime(
-+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
-+ )))
-+
-+ # Verify the database
-+ with open(public_key, "r") as f:
-+ if not db.verify(f):
-+ log.error("Could not verify database")
-+ return False
-+
-+ return True
-diff --git a/src/python/location/export.py b/src/python/location/export.py
-new file mode 100644
-index 0000000..942700e
---- /dev/null
-+++ b/src/python/location/export.py
-@@ -0,0 +1,291 @@
-+#!/usr/bin/python3
-+###############################################################################
-+# #
-+# libloc - A library to determine the location of someone on the Internet #
-+# #
-+# Copyright (C) 2020-2021 IPFire Development Team <info@ipfire.org> #
-+# #
-+# This library is free software; you can redistribute it and/or #
-+# modify it under the terms of the GNU Lesser General Public #
-+# License as published by the Free Software Foundation; either #
-+# version 2.1 of the License, or (at your option) any later version. #
-+# #
-+# This library is distributed in the hope that it will be useful, #
-+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
-+# Lesser General Public License for more details. #
-+# #
-+###############################################################################
-+
-+import io
-+import ipaddress
-+import logging
-+import math
-+import os
-+import socket
-+import sys
-+
-+from .i18n import _
-+import _location
-+
-+# Initialise logging
-+log = logging.getLogger("location.export")
-+log.propagate = 1
-+
-+FLAGS = {
-+ _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
-+ _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
-+ _location.NETWORK_FLAG_ANYCAST : "A3",
-+ _location.NETWORK_FLAG_DROP : "XD",
-+}
-+
-+class OutputWriter(object):
-+ suffix = "networks"
-+ mode = "w"
-+
-+ def __init__(self, name, family=None, directory=None, f=None):
-+ self.name = name
-+ self.family = family
-+ self.directory = directory
-+
-+ # Open output file
-+ if f:
-+ self.f = f
-+ elif self.directory:
-+ self.f = open(self.filename, self.mode)
-+ elif "b" in self.mode:
-+ self.f = io.BytesIO()
-+ else:
-+ self.f = io.StringIO()
-+
-+ # Tag
-+ self.tag = self._make_tag()
-+
-+ # Call any custom initialization
-+ self.init()
-+
-+ # Immediately write the header
-+ self._write_header()
-+
-+ def init(self):
-+ """
-+ To be overwritten by anything that inherits from this
-+ """
-+ pass
-+
-+ def __repr__(self):
-+ return "<%s %s f=%s>" % (self.__class__.__name__, self, self.f)
-+
-+ def _make_tag(self):
-+ families = {
-+ socket.AF_INET6 : "6",
-+ socket.AF_INET : "4",
-+ }
-+
-+ return "%sv%s" % (self.name, families.get(self.family, "?"))
-+
-+ def filename(self):
-+ if self.directory:
-+ return os.path.join(self.directory, "%s.%s" % (self.tag, self.suffix))
-+
-+ def _write_header(self):
-+ """
-+ The header of the file
-+ """
-+ pass
-+
-+ def _write_footer(self):
-+ """
-+ The footer of the file
-+ """
-+ pass
-+
-+ def write(self, network):
-+ self.f.write("%s\n" % network)
-+
-+ def finish(self):
-+ """
-+ Called when all data has been written
-+ """
-+ self._write_footer()
-+
-+ # Flush all output
-+ self.f.flush()
-+
-+ def print(self):
-+ """
-+ Prints the entire output line by line
-+ """
-+ if isinstance(self.f, io.BytesIO):
-+ raise TypeError(_("Won't write binary output to stdout"))
-+
-+ # Go back to the beginning
-+ self.f.seek(0)
-+
-+ # Iterate over everything line by line
-+ for line in self.f:
-+ sys.stdout.write(line)
-+
-+
-+class IpsetOutputWriter(OutputWriter):
-+ """
-+ For ipset
-+ """
-+ suffix = "ipset"
-+
-+ # The value is being used if we don't know any better
-+ DEFAULT_HASHSIZE = 64
-+
-+ # We aim for this many networks in a bucket on average. This allows us to choose
-+ # how much memory we want to sacrifice to gain better performance. The lower the
-+ # factor, the faster a lookup will be, but it will use more memory.
-+ # We will aim for only using three quarters of all buckets to avoid any searches
-+ # through the linked lists.
-+ HASHSIZE_FACTOR = 0.75
-+
-+ def init(self):
-+ # Count all networks
-+ self.networks = 0
-+
-+ @property
-+ def hashsize(self):
-+ """
-+ Calculates an optimized hashsize
-+ """
-+ # Return the default value if we don't know the size of the set
-+ if not self.networks:
-+ return self.DEFAULT_HASHSIZE
-+
-+ # Find the nearest power of two that is larger than the number of networks
-+ # divided by the hashsize factor.
-+ exponent = math.log(self.networks / self.HASHSIZE_FACTOR, 2)
-+
-+ # Return the size of the hash (the minimum is 64)
-+ return max(2 ** math.ceil(exponent), 64)
-+
-+ def _write_header(self):
-+ # This must have a fixed size, because we will write the header again in the end
-+ self.f.write("create %s hash:net family inet%s" % (
-+ self.tag,
-+ "6" if self.family == socket.AF_INET6 else ""
-+ ))
-+ self.f.write(" hashsize %8d maxelem 1048576 -exist\n" % self.hashsize)
-+ self.f.write("flush %s\n" % self.tag)
-+
-+ def write(self, network):
-+ self.f.write("add %s %s\n" % (self.tag, network))
-+
-+ # Increment network counter
-+ self.networks += 1
-+
-+ def _write_footer(self):
-+ # Jump back to the beginning of the file
-+ self.f.seek(0)
-+
-+ # Rewrite the header with better configuration
-+ self._write_header()
-+
-+
-+class NftablesOutputWriter(OutputWriter):
-+ """
-+ For nftables
-+ """
-+ suffix = "set"
-+
-+ def _write_header(self):
-+ self.f.write("define %s = {\n" % self.tag)
-+
-+ def _write_footer(self):
-+ self.f.write("}\n")
-+
-+ def write(self, network):
-+ self.f.write(" %s,\n" % network)
-+
-+
-+class XTGeoIPOutputWriter(OutputWriter):
-+ """
-+ Formats the output in that way, that it can be loaded by
-+ the xt_geoip kernel module from xtables-addons.
-+ """
-+ mode = "wb"
-+
-+ @property
-+ def tag(self):
-+ return self.name
-+
-+ @property
-+ def suffix(self):
-+ return "iv%s" % ("6" if self.family == socket.AF_INET6 else "4")
-+
-+ def write(self, network):
-+ self.f.write(network._first_address)
-+ self.f.write(network._last_address)
-+
-+
-+formats = {
-+ "ipset" : IpsetOutputWriter,
-+ "list" : OutputWriter,
-+ "nftables" : NftablesOutputWriter,
-+ "xt_geoip" : XTGeoIPOutputWriter,
-+}
-+
-+class Exporter(object):
-+ def __init__(self, db, writer):
-+ self.db, self.writer = db, writer
-+
-+ def export(self, directory, families, countries, asns):
-+ for family in families:
-+ log.debug("Exporting family %s" % family)
-+
-+ writers = {}
-+
-+ # Create writers for countries
-+ for country_code in countries:
-+ writers[country_code] = self.writer(country_code, family=family, directory=directory)
-+
-+ # Create writers for ASNs
-+ for asn in asns:
-+ writers[asn] = self.writer("AS%s" % asn, family=family, directory=directory)
-+
-+ # Filter countries from special country codes
-+ country_codes = [
-+ country_code for country_code in countries if not country_code in FLAGS.values()
-+ ]
-+
-+ # Get all networks that match the family
-+ networks = self.db.search_networks(family=family,
-+ country_codes=country_codes, asns=asns, flatten=True)
-+
-+ # Walk through all networks
-+ for network in networks:
-+ # Write matching countries
-+ try:
-+ writers[network.country_code].write(network)
-+ except KeyError:
-+ pass
-+
-+ # Write matching ASNs
-+ try:
-+ writers[network.asn].write(network)
-+ except KeyError:
-+ pass
-+
-+ # Handle flags
-+ for flag in FLAGS:
-+ if network.has_flag(flag):
-+ # Fetch the "fake" country code
-+ country = FLAGS[flag]
-+
-+ try:
-+ writers[country].write(network)
-+ except KeyError:
-+ pass
-+
-+ # Write everything to the filesystem
-+ for writer in writers.values():
-+ writer.finish()
-+
-+ # Print to stdout
-+ if not directory:
-+ for writer in writers.values():
-+ writer.print()
-diff --git a/src/python/location/i18n.py b/src/python/location/i18n.py
-new file mode 100644
-index 0000000..2161aa6
---- /dev/null
-+++ b/src/python/location/i18n.py
-@@ -0,0 +1,26 @@
-+#!/usr/bin/python3
-+###############################################################################
-+# #
-+# libloc - A library to determine the location of someone on the Internet #
-+# #
-+# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
-+# #
-+# This library is free software; you can redistribute it and/or #
-+# modify it under the terms of the GNU Lesser General Public #
-+# License as published by the Free Software Foundation; either #
-+# version 2.1 of the License, or (at your option) any later version. #
-+# #
-+# This library is distributed in the hope that it will be useful, #
-+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
-+# Lesser General Public License for more details. #
-+# #
-+###############################################################################
-+
-+import gettext
-+
-+def _(singular, plural=None, n=None):
-+ if plural:
-+ return gettext.dngettext("libloc", singular, plural, n)
-+
-+ return gettext.dgettext("libloc", singular)
-diff --git a/src/python/location/importer.py b/src/python/location/importer.py
-new file mode 100644
-index 0000000..dee36ed
---- /dev/null
-+++ b/src/python/location/importer.py
-@@ -0,0 +1,250 @@
-+#!/usr/bin/python3
-+###############################################################################
-+# #
-+# libloc - A library to determine the location of someone on the Internet #
-+# #
-+# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
-+# #
-+# This library is free software; you can redistribute it and/or #
-+# modify it under the terms of the GNU Lesser General Public #
-+# License as published by the Free Software Foundation; either #
-+# version 2.1 of the License, or (at your option) any later version. #
-+# #
-+# This library is distributed in the hope that it will be useful, #
-+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
-+# Lesser General Public License for more details. #
-+# #
-+###############################################################################
-+
-+import gzip
-+import logging
-+import urllib.request
-+
-+# Initialise logging
-+log = logging.getLogger("location.importer")
-+log.propagate = 1
-+
-+WHOIS_SOURCES = {
-+ # African Network Information Centre
-+ "AFRINIC": [
-+ "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz"
-+ ],
-+
-+ # Asia Pacific Network Information Centre
-+ "APNIC": [
-+ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-+ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
-+ #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
-+ #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
-+ "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
-+ "https://ftp.apnic.net/apnic/whois/apnic.db.organisation.gz"
-+ ],
-+
-+ # American Registry for Internet Numbers
-+ # XXX there is nothing useful for us in here
-+ # ARIN: [
-+ # "https://ftp.arin.net/pub/rr/arin.db"
-+ # ],
-+
-+ # Japan Network Information Center
-+ "JPNIC": [
-+ "https://ftp.nic.ad.jp/jpirr/jpirr.db.gz"
-+ ],
-+
-+ # Latin America and Caribbean Network Information Centre
-+ "LACNIC": [
-+ "https://ftp.lacnic.net/lacnic/dbase/lacnic.db.gz"
-+ ],
-+
-+ # Réseaux IP Européens
-+ "RIPE": [
-+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
-+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
-+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
-+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
-+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.organisation.gz"
-+ ],
-+}
-+
-+EXTENDED_SOURCES = {
-+ # African Network Information Centre
-+ # "ARIN": [
-+ # "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest"
-+ # ],
-+
-+ # Asia Pacific Network Information Centre
-+ # "APNIC": [
-+ # "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest"
-+ # ],
-+
-+ # American Registry for Internet Numbers
-+ "ARIN": [
-+ "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest"
-+ ],
-+
-+ # Latin America and Caribbean Network Information Centre
-+ "LACNIC": [
-+ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest"
-+ ],
-+
-+ # Réseaux IP Européens
-+ # "RIPE": [
-+ # "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest"
-+ # ],
-+}
-+
-+class Downloader(object):
-+ def __init__(self):
-+ self.proxy = None
-+
-+ def set_proxy(self, url):
-+ """
-+ Sets a HTTP proxy that is used to perform all requests
-+ """
-+ log.info("Using proxy %s" % url)
-+ self.proxy = url
-+
-+ def request(self, url, data=None, return_blocks=False):
-+ req = urllib.request.Request(url, data=data)
-+
-+ # Configure proxy
-+ if self.proxy:
-+ req.set_proxy(self.proxy, "http")
-+
-+ return DownloaderContext(self, req, return_blocks=return_blocks)
-+
-+
-+class DownloaderContext(object):
-+ def __init__(self, downloader, request, return_blocks=False):
-+ self.downloader = downloader
-+ self.request = request
-+
-+ # Should we return one block or a single line?
-+ self.return_blocks = return_blocks
-+
-+ # Save the response object
-+ self.response = None
-+
-+ def __enter__(self):
-+ log.info("Retrieving %s..." % self.request.full_url)
-+
-+ # Send request
-+ self.response = urllib.request.urlopen(self.request)
-+
-+ # Log the response headers
-+ log.debug("Response Headers:")
-+ for header in self.headers:
-+ log.debug(" %s: %s" % (header, self.get_header(header)))
-+
-+ return self
-+
-+ def __exit__(self, type, value, traceback):
-+ pass
-+
-+ def __iter__(self):
-+ """
-+ Makes the object iterable by going through each block
-+ """
-+ if self.return_blocks:
-+ return iterate_over_blocks(self.body)
-+
-+ return iterate_over_lines(self.body)
-+
-+ @property
-+ def headers(self):
-+ if self.response:
-+ return self.response.headers
-+
-+ def get_header(self, name):
-+ if self.headers:
-+ return self.headers.get(name)
-+
-+ @property
-+ def body(self):
-+ """
-+ Returns a file-like object with the decoded content
-+ of the response.
-+ """
-+ content_type = self.get_header("Content-Type")
-+
-+ # Decompress any gzipped response on the fly
-+ if content_type in ("application/x-gzip", "application/gzip"):
-+ return gzip.GzipFile(fileobj=self.response, mode="rb")
-+
-+ # Return the response by default
-+ return self.response
-+
-+
-+def read_blocks(f):
-+ for block in iterate_over_blocks(f):
-+ type = None
-+ data = {}
-+
-+ for i, line in enumerate(block):
-+ key, value = line.split(":", 1)
-+
-+ # The key of the first line defines the type
-+ if i == 0:
-+ type = key
-+
-+ # Store value
-+ data[key] = value.strip()
-+
-+ yield type, data
-+
-+def iterate_over_blocks(f, charsets=("utf-8", "latin1")):
-+ block = []
-+
-+ for line in f:
-+ # Convert to string
-+ for charset in charsets:
-+ try:
-+ line = line.decode(charset)
-+ except UnicodeDecodeError:
-+ continue
-+ else:
-+ break
-+
-+ # Skip commented lines
-+ if line.startswith("#") or line.startswith("%"):
-+ continue
-+
-+ # Strip line-endings
-+ line = line.rstrip()
-+
-+ # Remove any comments at the end of line
-+ line, hash, comment = line.partition("#")
-+
-+ if comment:
-+ # Strip any whitespace before the comment
-+ line = line.rstrip()
-+
-+ # If the line is now empty, we move on
-+ if not line:
-+ continue
-+
-+ if line:
-+ block.append(line)
-+ continue
-+
-+ # End the block on an empty line
-+ if block:
-+ yield block
-+
-+ # Reset the block
-+ block = []
-+
-+ # Return the last block
-+ if block:
-+ yield block
-+
-+
-+def iterate_over_lines(f):
-+ for line in f:
-+ # Decode the line
-+ line = line.decode()
-+
-+ # Strip the ending
-+ yield line.rstrip()
-diff --git a/src/python/location/logger.py b/src/python/location/logger.py
-new file mode 100644
-index 0000000..0bdf9ec
---- /dev/null
-+++ b/src/python/location/logger.py
-@@ -0,0 +1,46 @@
-+#!/usr/bin/python3
-+###############################################################################
-+# #
-+# libloc - A library to determine the location of someone on the Internet #
-+# #
-+# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
-+# #
-+# This library is free software; you can redistribute it and/or #
-+# modify it under the terms of the GNU Lesser General Public #
-+# License as published by the Free Software Foundation; either #
-+# version 2.1 of the License, or (at your option) any later version. #
-+# #
-+# This library is distributed in the hope that it will be useful, #
-+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
-+# Lesser General Public License for more details. #
-+# #
-+###############################################################################
-+
-+import logging
-+import logging.handlers
-+
-+# Initialise root logger
-+log = logging.getLogger("location")
-+log.setLevel(logging.INFO)
-+
-+# Log to console
-+handler = logging.StreamHandler()
-+handler.setLevel(logging.DEBUG)
-+log.addHandler(handler)
-+
-+# Log to syslog
-+handler = logging.handlers.SysLogHandler(address="/dev/log",
-+ facility=logging.handlers.SysLogHandler.LOG_DAEMON)
-+handler.setLevel(logging.INFO)
-+log.addHandler(handler)
-+
-+# Format syslog messages
-+formatter = logging.Formatter("%(message)s")
-+handler.setFormatter(formatter)
-+
-+def set_level(level):
-+ """
-+ Sets the log level for the root logger
-+ """
-+ log.setLevel(level)
-diff --git a/src/python/logger.py b/src/python/logger.py
-deleted file mode 100644
-index 0bdf9ec..0000000
---- a/src/python/logger.py
-+++ /dev/null
-@@ -1,46 +0,0 @@
--#!/usr/bin/python3
--###############################################################################
--# #
--# libloc - A library to determine the location of someone on the Internet #
--# #
--# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
--# #
--# This library is free software; you can redistribute it and/or #
--# modify it under the terms of the GNU Lesser General Public #
--# License as published by the Free Software Foundation; either #
--# version 2.1 of the License, or (at your option) any later version. #
--# #
--# This library is distributed in the hope that it will be useful, #
--# but WITHOUT ANY WARRANTY; without even the implied warranty of #
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
--# Lesser General Public License for more details. #
--# #
--###############################################################################
--
--import logging
--import logging.handlers
--
--# Initialise root logger
--log = logging.getLogger("location")
--log.setLevel(logging.INFO)
--
--# Log to console
--handler = logging.StreamHandler()
--handler.setLevel(logging.DEBUG)
--log.addHandler(handler)
--
--# Log to syslog
--handler = logging.handlers.SysLogHandler(address="/dev/log",
-- facility=logging.handlers.SysLogHandler.LOG_DAEMON)
--handler.setLevel(logging.INFO)
--log.addHandler(handler)
--
--# Format syslog messages
--formatter = logging.Formatter("%(message)s")
--handler.setFormatter(formatter)
--
--def set_level(level):
-- """
-- Sets the log level for the root logger
-- """
-- log.setLevel(level)
-diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in
-new file mode 100644
-index 0000000..bee9186
---- /dev/null
-+++ b/src/scripts/location-importer.in
-@@ -0,0 +1,1535 @@
-+#!/usr/bin/python3
-+###############################################################################
-+# #
-+# libloc - A library to determine the location of someone on the Internet #
-+# #
-+# Copyright (C) 2020-2022 IPFire Development Team <info@ipfire.org> #
-+# #
-+# This library is free software; you can redistribute it and/or #
-+# modify it under the terms of the GNU Lesser General Public #
-+# License as published by the Free Software Foundation; either #
-+# version 2.1 of the License, or (at your option) any later version. #
-+# #
-+# This library is distributed in the hope that it will be useful, #
-+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
-+# Lesser General Public License for more details. #
-+# #
-+###############################################################################
-+
-+import argparse
-+import ipaddress
-+import json
-+import logging
-+import math
-+import re
-+import socket
-+import sys
-+import telnetlib
-+
-+# Load our location module
-+import location
-+import location.database
-+import location.importer
-+from location.i18n import _
-+
-+# Initialise logging
-+log = logging.getLogger("location.importer")
-+log.propagate = 1
-+
-+# Define constants
-+VALID_ASN_RANGES = (
-+ (1, 23455),
-+ (23457, 64495),
-+ (131072, 4199999999),
-+)
-+
-+
-+class CLI(object):
-+ def parse_cli(self):
-+ parser = argparse.ArgumentParser(
-+ description=_("Location Importer Command Line Interface"),
-+ )
-+ subparsers = parser.add_subparsers()
-+
-+ # Global configuration flags
-+ parser.add_argument("--debug", action="store_true",
-+ help=_("Enable debug output"))
-+ parser.add_argument("--quiet", action="store_true",
-+ help=_("Enable quiet mode"))
-+
-+ # version
-+ parser.add_argument("--version", action="version",
-+ version="%(prog)s @VERSION@")
-+
-+ # Database
-+ parser.add_argument("--database-host", required=True,
-+ help=_("Database Hostname"), metavar=_("HOST"))
-+ parser.add_argument("--database-name", required=True,
-+ help=_("Database Name"), metavar=_("NAME"))
-+ parser.add_argument("--database-username", required=True,
-+ help=_("Database Username"), metavar=_("USERNAME"))
-+ parser.add_argument("--database-password", required=True,
-+ help=_("Database Password"), metavar=_("PASSWORD"))
-+
-+ # Write Database
-+ write = subparsers.add_parser("write", help=_("Write database to file"))
-+ write.set_defaults(func=self.handle_write)
-+ write.add_argument("file", nargs=1, help=_("Database File"))
-+ write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
-+ write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
-+ write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
-+ write.add_argument("--description", nargs="?", help=_("Sets a description"))
-+ write.add_argument("--license", nargs="?", help=_("Sets the license"))
-+ write.add_argument("--version", type=int, help=_("Database Format Version"))
-+
-+ # Update WHOIS
-+ update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
-+ update_whois.set_defaults(func=self.handle_update_whois)
-+
-+ # Update announcements
-+ update_announcements = subparsers.add_parser("update-announcements",
-+ help=_("Update BGP Annoucements"))
-+ update_announcements.set_defaults(func=self.handle_update_announcements)
-+ update_announcements.add_argument("server", nargs=1,
-+ help=_("Route Server to connect to"), metavar=_("SERVER"))
-+
-+ # Update overrides
-+ update_overrides = subparsers.add_parser("update-overrides",
-+ help=_("Update overrides"),
-+ )
-+ update_overrides.add_argument(
-+ "files", nargs="+", help=_("Files to import"),
-+ )
-+ update_overrides.set_defaults(func=self.handle_update_overrides)
-+
-+ # Import countries
-+ import_countries = subparsers.add_parser("import-countries",
-+ help=_("Import countries"),
-+ )
-+ import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
-+ help=_("File to import"))
-+ import_countries.set_defaults(func=self.handle_import_countries)
-+
-+ args = parser.parse_args()
-+
-+ # Configure logging
-+ if args.debug:
-+ location.logger.set_level(logging.DEBUG)
-+ elif args.quiet:
-+ location.logger.set_level(logging.WARNING)
-+
-+ # Print usage if no action was given
-+ if not "func" in args:
-+ parser.print_usage()
-+ sys.exit(2)
-+
-+ return args
-+
-+ def run(self):
-+ # Parse command line arguments
-+ args = self.parse_cli()
-+
-+ # Initialise database
-+ self.db = self._setup_database(args)
-+
-+ # Call function
-+ ret = args.func(args)
-+
-+ # Return with exit code
-+ if ret:
-+ sys.exit(ret)
-+
-+ # Otherwise just exit
-+ sys.exit(0)
-+
-+ def _setup_database(self, ns):
-+ """
-+ Initialise the database
-+ """
-+ # Connect to database
-+ db = location.database.Connection(
-+ host=ns.database_host, database=ns.database_name,
-+ user=ns.database_username, password=ns.database_password,
-+ )
-+
-+ with db.transaction():
-+ db.execute("""
-+ -- announcements
-+ CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
-+ first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
-+ last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
-+ CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
-+ CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
-+ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
-+
-+ -- autnums
-+ CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
-+ ALTER TABLE autnums ADD COLUMN IF NOT EXISTS source text;
-+ CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
-+
-+ -- countries
-+ CREATE TABLE IF NOT EXISTS countries(
-+ country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
-+ CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
-+
-+ -- networks
-+ CREATE TABLE IF NOT EXISTS networks(network inet, country text);
-+ ALTER TABLE networks ADD COLUMN IF NOT EXISTS original_countries text[];
-+ ALTER TABLE networks ADD COLUMN IF NOT EXISTS source text;
-+ CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
-+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
-+ CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
-+
-+ -- overrides
-+ CREATE TABLE IF NOT EXISTS autnum_overrides(
-+ number bigint NOT NULL,
-+ name text,
-+ country text,
-+ is_anonymous_proxy boolean,
-+ is_satellite_provider boolean,
-+ is_anycast boolean
-+ );
-+ CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
-+ ON autnum_overrides(number);
-+ ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS source text;
-+ ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
-+
-+ CREATE TABLE IF NOT EXISTS network_overrides(
-+ network inet NOT NULL,
-+ country text,
-+ is_anonymous_proxy boolean,
-+ is_satellite_provider boolean,
-+ is_anycast boolean
-+ );
-+ CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
-+ ON network_overrides(network);
-+ CREATE INDEX IF NOT EXISTS network_overrides_search
-+ ON network_overrides USING GIST(network inet_ops);
-+ ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS source text;
-+ ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
-+ """)
-+
-+ return db
-+
-+ def handle_write(self, ns):
-+ """
-+ Compiles a database in libloc format out of what is in the database
-+ """
-+ # Allocate a writer
-+ writer = location.Writer(ns.signing_key, ns.backup_signing_key)
-+
-+ # Set all metadata
-+ if ns.vendor:
-+ writer.vendor = ns.vendor
-+
-+ if ns.description:
-+ writer.description = ns.description
-+
-+ if ns.license:
-+ writer.license = ns.license
-+
-+ # Add all Autonomous Systems
-+ log.info("Writing Autonomous Systems...")
-+
-+ # Select all ASes with a name
-+ rows = self.db.query("""
-+ SELECT
-+ autnums.number AS number,
-+ COALESCE(
-+ (SELECT overrides.name FROM autnum_overrides overrides
-+ WHERE overrides.number = autnums.number),
-+ autnums.name
-+ ) AS name
-+ FROM autnums
-+ WHERE name <> %s ORDER BY number
-+ """, "")
-+
-+ for row in rows:
-+ a = writer.add_as(row.number)
-+ a.name = row.name
-+
-+ # Add all networks
-+ log.info("Writing networks...")
-+
-+ # Select all known networks
-+ rows = self.db.query("""
-+ WITH known_networks AS (
-+ SELECT network FROM announcements
-+ UNION
-+ SELECT network FROM networks
-+ UNION
-+ SELECT network FROM network_overrides
-+ ),
-+
-+ ordered_networks AS (
-+ SELECT
-+ known_networks.network AS network,
-+ announcements.autnum AS autnum,
-+ networks.country AS country,
-+
-+ -- Must be part of returned values for ORDER BY clause
-+ masklen(announcements.network) AS sort_a,
-+ masklen(networks.network) AS sort_b
-+ FROM
-+ known_networks
-+ LEFT JOIN
-+ announcements ON known_networks.network <<= announcements.network
-+ LEFT JOIN
-+ networks ON known_networks.network <<= networks.network
-+ ORDER BY
-+ known_networks.network,
-+ sort_a DESC,
-+ sort_b DESC
-+ )
-+
-+ -- Return a list of those networks enriched with all
-+ -- other information that we store in the database
-+ SELECT
-+ DISTINCT ON (network)
-+ network,
-+ autnum,
-+
-+ -- Country
-+ COALESCE(
-+ (
-+ SELECT country FROM network_overrides overrides
-+ WHERE networks.network <<= overrides.network
-+ ORDER BY masklen(overrides.network) DESC
-+ LIMIT 1
-+ ),
-+ (
-+ SELECT country FROM autnum_overrides overrides
-+ WHERE networks.autnum = overrides.number
-+ ),
-+ networks.country
-+ ) AS country,
-+
-+ -- Flags
-+ COALESCE(
-+ (
-+ SELECT is_anonymous_proxy FROM network_overrides overrides
-+ WHERE networks.network <<= overrides.network
-+ ORDER BY masklen(overrides.network) DESC
-+ LIMIT 1
-+ ),
-+ (
-+ SELECT is_anonymous_proxy FROM autnum_overrides overrides
-+ WHERE networks.autnum = overrides.number
-+ ),
-+ FALSE
-+ ) AS is_anonymous_proxy,
-+ COALESCE(
-+ (
-+ SELECT is_satellite_provider FROM network_overrides overrides
-+ WHERE networks.network <<= overrides.network
-+ ORDER BY masklen(overrides.network) DESC
-+ LIMIT 1
-+ ),
-+ (
-+ SELECT is_satellite_provider FROM autnum_overrides overrides
-+ WHERE networks.autnum = overrides.number
-+ ),
-+ FALSE
-+ ) AS is_satellite_provider,
-+ COALESCE(
-+ (
-+ SELECT is_anycast FROM network_overrides overrides
-+ WHERE networks.network <<= overrides.network
-+ ORDER BY masklen(overrides.network) DESC
-+ LIMIT 1
-+ ),
-+ (
-+ SELECT is_anycast FROM autnum_overrides overrides
-+ WHERE networks.autnum = overrides.number
-+ ),
-+ FALSE
-+ ) AS is_anycast,
-+ COALESCE(
-+ (
-+ SELECT is_drop FROM network_overrides overrides
-+ WHERE networks.network <<= overrides.network
-+ ORDER BY masklen(overrides.network) DESC
-+ LIMIT 1
-+ ),
-+ (
-+ SELECT is_drop FROM autnum_overrides overrides
-+ WHERE networks.autnum = overrides.number
-+ ),
-+ FALSE
-+ ) AS is_drop
-+ FROM
-+ ordered_networks networks
-+ """)
-+
-+ for row in rows:
-+ network = writer.add_network(row.network)
-+
-+ # Save country
-+ if row.country:
-+ network.country_code = row.country
-+
-+ # Save ASN
-+ if row.autnum:
-+ network.asn = row.autnum
-+
-+ # Set flags
-+ if row.is_anonymous_proxy:
-+ network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
-+
-+ if row.is_satellite_provider:
-+ network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
-+
-+ if row.is_anycast:
-+ network.set_flag(location.NETWORK_FLAG_ANYCAST)
-+
-+ if row.is_drop:
-+ network.set_flag(location.NETWORK_FLAG_DROP)
-+
-+ # Add all countries
-+ log.info("Writing countries...")
-+ rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
-+
-+ for row in rows:
-+ c = writer.add_country(row.country_code)
-+ c.continent_code = row.continent_code
-+ c.name = row.name
-+
-+ # Write everything to file
-+ log.info("Writing database to file...")
-+ for file in ns.file:
-+ writer.write(file)
-+
-+ def handle_update_whois(self, ns):
-+ downloader = location.importer.Downloader()
-+
-+ # Download all sources
-+ with self.db.transaction():
-+ # Create some temporary tables to store parsed data
-+ self.db.execute("""
-+ CREATE TEMPORARY TABLE _autnums(number integer NOT NULL, organization text NOT NULL, source text NOT NULL)
-+ ON COMMIT DROP;
-+ CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
-+
-+ CREATE TEMPORARY TABLE _organizations(handle text NOT NULL, name text NOT NULL, source text NOT NULL)
-+ ON COMMIT DROP;
-+ CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
-+
-+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL, original_countries text[] NOT NULL, source text NOT NULL)
-+ ON COMMIT DROP;
-+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
-+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
-+ """)
-+
-+ # Remove all previously imported content
-+ self.db.execute("""
-+ TRUNCATE TABLE networks;
-+ """)
-+
-+ # Fetch all valid country codes to check parsed networks aganist...
-+ rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
-+ validcountries = []
-+
-+ for row in rows:
-+ validcountries.append(row.country_code)
-+
-+ for source_key in location.importer.WHOIS_SOURCES:
-+ for single_url in location.importer.WHOIS_SOURCES[source_key]:
-+ with downloader.request(single_url, return_blocks=True) as f:
-+ for block in f:
-+ self._parse_block(block, source_key, validcountries)
-+
-+ # Process all parsed networks from every RIR we happen to have access to,
-+ # insert the largest network chunks into the networks table immediately...
-+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
-+
-+ for family in (row.family for row in families):
-+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
-+
-+ self.db.execute("INSERT INTO networks(network, country, original_countries, source) \
-+ SELECT network, country, original_countries, source FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
-+
-+ # ... determine any other prefixes for this network family, ...
-+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
-+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
-+
-+ # ... and insert networks with this prefix in case they provide additional
-+ # information (i. e. subnet of a larger chunk with a different country)
-+ for prefix in (row.prefix for row in prefixes):
-+ self.db.execute("""
-+ WITH candidates AS (
-+ SELECT
-+ _rirdata.network,
-+ _rirdata.country,
-+ _rirdata.original_countries,
-+ _rirdata.source
-+ FROM
-+ _rirdata
-+ WHERE
-+ family(_rirdata.network) = %s
-+ AND
-+ masklen(_rirdata.network) = %s
-+ ),
-+ filtered AS (
-+ SELECT
-+ DISTINCT ON (c.network)
-+ c.network,
-+ c.country,
-+ c.original_countries,
-+ c.source,
-+ masklen(networks.network),
-+ networks.country AS parent_country
-+ FROM
-+ candidates c
-+ LEFT JOIN
-+ networks
-+ ON
-+ c.network << networks.network
-+ ORDER BY
-+ c.network,
-+ masklen(networks.network) DESC NULLS LAST
-+ )
-+ INSERT INTO
-+ networks(network, country, original_countries, source)
-+ SELECT
-+ network,
-+ country,
-+ original_countries,
-+ source
-+ FROM
-+ filtered
-+ WHERE
-+ parent_country IS NULL
-+ OR
-+ country <> parent_country
-+ ON CONFLICT DO NOTHING""",
-+ family, prefix,
-+ )
-+
-+ self.db.execute("""
-+ INSERT INTO autnums(number, name, source)
-+ SELECT _autnums.number, _organizations.name, _organizations.source FROM _autnums
-+ JOIN _organizations ON _autnums.organization = _organizations.handle
-+ ON CONFLICT (number) DO UPDATE SET name = excluded.name;
-+ """)
-+
-+ # Download all extended sources
-+ for source_key in location.importer.EXTENDED_SOURCES:
-+ for single_url in location.importer.EXTENDED_SOURCES[source_key]:
-+ with self.db.transaction():
-+ # Download data
-+ with downloader.request(single_url) as f:
-+ for line in f:
-+ self._parse_line(line, source_key, validcountries)
-+
-+ # Download and import (technical) AS names from ARIN
-+ self._import_as_names_from_arin()
-+
-+ def _check_parsed_network(self, network):
-+ """
-+ Assistive function to detect and subsequently sort out parsed
-+ networks from RIR data (both Whois and so-called "extended sources"),
-+ which are or have...
-+
-+ (a) not globally routable (RFC 1918 space, et al.)
-+ (b) covering a too large chunk of the IP address space (prefix length
-+ is < 7 for IPv4 networks, and < 10 for IPv6)
-+ (c) "0.0.0.0" or "::" as a network address
-+ (d) are too small for being publicly announced (we have decided not to
-+ process them at the moment, as they significantly enlarge our
-+ database without providing very helpful additional information)
-+
-+ This unfortunately is necessary due to brain-dead clutter across
-+ various RIR databases, causing mismatches and eventually disruptions.
-+
-+ We will return False in case a network is not suitable for adding
-+ it to our database, and True otherwise.
-+ """
-+
-+ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
-+ return False
-+
-+ if not network.is_global:
-+ log.debug("Skipping non-globally routable network: %s" % network)
-+ return False
-+
-+ if network.version == 4:
-+ if network.prefixlen < 7:
-+ log.debug("Skipping too big IP chunk: %s" % network)
-+ return False
-+
-+ if network.prefixlen > 24:
-+ log.debug("Skipping network too small to be publicly announced: %s" % network)
-+ return False
-+
-+ if str(network.network_address) == "0.0.0.0":
-+ log.debug("Skipping network based on 0.0.0.0: %s" % network)
-+ return False
-+
-+ elif network.version == 6:
-+ if network.prefixlen < 10:
-+ log.debug("Skipping too big IP chunk: %s" % network)
-+ return False
-+
-+ if network.prefixlen > 48:
-+ log.debug("Skipping network too small to be publicly announced: %s" % network)
-+ return False
-+
-+ if str(network.network_address) == "::":
-+ log.debug("Skipping network based on '::': %s" % network)
-+ return False
-+
-+ else:
-+ # This should not happen...
-+ log.warning("Skipping network of unknown family, this should not happen: %s" % network)
-+ return False
-+
-+ # In case we have made it here, the network is considered to
-+ # be suitable for libloc consumption...
-+ return True
-+
-+ def _check_parsed_asn(self, asn):
-+ """
-+ Assistive function to filter Autonomous System Numbers not being suitable
-+ for adding to our database. Returns False in such cases, and True otherwise.
-+ """
-+
-+ for start, end in VALID_ASN_RANGES:
-+ if start <= asn and end >= asn:
-+ return True
-+
-+ log.info("Supplied ASN %s out of publicly routable ASN ranges" % asn)
-+ return False
-+
-+ def _parse_block(self, block, source_key, validcountries = None):
-+ # Get first line to find out what type of block this is
-+ line = block[0]
-+
-+ # aut-num
-+ if line.startswith("aut-num:"):
-+ return self._parse_autnum_block(block, source_key)
-+
-+ # inetnum
-+ if line.startswith("inet6num:") or line.startswith("inetnum:"):
-+ return self._parse_inetnum_block(block, source_key, validcountries)
-+
-+ # organisation
-+ elif line.startswith("organisation:"):
-+ return self._parse_org_block(block, source_key)
-+
-+ def _parse_autnum_block(self, block, source_key):
-+ autnum = {}
-+ for line in block:
-+ # Split line
-+ key, val = split_line(line)
-+
-+ if key == "aut-num":
-+ m = re.match(r"^(AS|as)(\d+)", val)
-+ if m:
-+ autnum["asn"] = m.group(2)
-+
-+ elif key == "org":
-+ autnum[key] = val.upper()
-+
-+ elif key == "descr":
-+ # Save the first description line as well...
-+ if not key in autnum:
-+ autnum[key] = val
-+
-+ # Skip empty objects
-+ if not autnum or not "asn" in autnum:
-+ return
-+
-+ # Insert a dummy organisation handle into our temporary organisations
-+ # table in case the AS does not have an organisation handle set, but
-+ # has a description (a quirk often observed in APNIC area), so we can
-+ # later display at least some string for this AS.
-+ if not "org" in autnum:
-+ if "descr" in autnum:
-+ autnum["org"] = "LIBLOC-%s-ORGHANDLE" % autnum.get("asn")
-+
-+ self.db.execute("INSERT INTO _organizations(handle, name, source) \
-+ VALUES(%s, %s, %s) ON CONFLICT (handle) DO NOTHING",
-+ autnum.get("org"), autnum.get("descr"), source_key,
-+ )
-+ else:
-+ log.warning("ASN %s neither has an organisation handle nor a description line set, omitting" % \
-+ autnum.get("asn"))
-+ return
-+
-+ # Insert into database
-+ self.db.execute("INSERT INTO _autnums(number, organization, source) \
-+ VALUES(%s, %s, %s) ON CONFLICT (number) DO UPDATE SET \
-+ organization = excluded.organization",
-+ autnum.get("asn"), autnum.get("org"), source_key,
-+ )
-+
-+ def _parse_inetnum_block(self, block, source_key, validcountries = None):
-+ log.debug("Parsing inetnum block:")
-+
-+ inetnum = {}
-+ for line in block:
-+ log.debug(line)
-+
-+ # Split line
-+ key, val = split_line(line)
-+
-+ # Filter any inetnum records which are only referring to IP space
-+ # not managed by that specific RIR...
-+ if key == "netname":
-+ if re.match(r"^(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK|STUB-[\d-]{3,}SLASH\d{1,2})", val.strip()):
-+ log.debug("Skipping record indicating historic/orphaned data: %s" % val.strip())
-+ return
-+
-+ if key == "inetnum":
-+ start_address, delim, end_address = val.partition("-")
-+
-+ # Strip any excess space
-+ start_address, end_address = start_address.rstrip(), end_address.strip()
-+
-+ # Handle "inetnum" formatting in LACNIC DB (e.g. "24.152.8/22" instead of "24.152.8.0/22")
-+ if start_address and not (delim or end_address):
-+ try:
-+ start_address = ipaddress.ip_network(start_address, strict=False)
-+ except ValueError:
-+ start_address = start_address.split("/")
-+ ldigits = start_address[0].count(".")
-+
-+ # How many octets do we need to add?
-+ # (LACNIC does not seem to have a /8 or greater assigned, so the following should suffice.)
-+ if ldigits == 1:
-+ start_address = start_address[0] + ".0.0/" + start_address[1]
-+ elif ldigits == 2:
-+ start_address = start_address[0] + ".0/" + start_address[1]
-+ else:
-+ log.warning("Could not recover IPv4 address from line in LACNIC DB format: %s" % line)
-+ return
-+
-+ try:
-+ start_address = ipaddress.ip_network(start_address, strict=False)
-+ except ValueError:
-+ log.warning("Could not parse line in LACNIC DB format: %s" % line)
-+ return
-+
-+ # Enumerate first and last IP address of this network
-+ end_address = start_address[-1]
-+ start_address = start_address[0]
-+
-+ else:
-+ # Convert to IP address
-+ try:
-+ start_address = ipaddress.ip_address(start_address)
-+ end_address = ipaddress.ip_address(end_address)
-+ except ValueError:
-+ log.warning("Could not parse line: %s" % line)
-+ return
-+
-+ inetnum["inetnum"] = list(ipaddress.summarize_address_range(start_address, end_address))
-+
-+ elif key == "inet6num":
-+ inetnum[key] = [ipaddress.ip_network(val, strict=False)]
-+
-+ elif key == "country":
-+ val = val.upper()
-+
-+ # Catch RIR data objects with more than one country code...
-+ if not key in inetnum:
-+ inetnum[key] = []
-+ else:
-+ if val in inetnum.get("country"):
-+ # ... but keep this list distinct...
-+ continue
-+
-+ # When people set country codes to "UK", they actually mean "GB"
-+ if val == "UK":
-+ val = "GB"
-+
-+ inetnum[key].append(val)
-+
-+ # Skip empty objects
-+ if not inetnum or not "country" in inetnum:
-+ return
-+
-+ # Prepare skipping objects with unknown country codes...
-+ invalidcountries = [singlecountry for singlecountry in inetnum.get("country") if singlecountry not in validcountries]
-+
-+ # Iterate through all networks enumerated from above, check them for plausibility and insert
-+ # them into the database, if _check_parsed_network() succeeded
-+ for single_network in inetnum.get("inet6num") or inetnum.get("inetnum"):
-+ if self._check_parsed_network(single_network):
-+
-+ # Skip objects with unknown country codes if they are valid to avoid log spam...
-+ if validcountries and invalidcountries:
-+ log.warning("Skipping network with bogus countr(y|ies) %s (original countries: %s): %s" % \
-+ (invalidcountries, inetnum.get("country"), inetnum.get("inet6num") or inetnum.get("inetnum")))
-+ break
-+
-+ # Everything is fine here, run INSERT statement...
-+ self.db.execute("INSERT INTO _rirdata(network, country, original_countries, source) \
-+ VALUES(%s, %s, %s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
-+ "%s" % single_network, inetnum.get("country")[0], inetnum.get("country"), source_key,
-+ )
-+
-+ def _parse_org_block(self, block, source_key):
-+ org = {}
-+ for line in block:
-+ # Split line
-+ key, val = split_line(line)
-+
-+ if key == "organisation":
-+ org[key] = val.upper()
-+ elif key == "org-name":
-+ org[key] = val
-+
-+ # Skip empty objects
-+ if not org:
-+ return
-+
-+ self.db.execute("INSERT INTO _organizations(handle, name, source) \
-+ VALUES(%s, %s, %s) ON CONFLICT (handle) DO \
-+ UPDATE SET name = excluded.name",
-+ org.get("organisation"), org.get("org-name"), source_key,
-+ )
-+
-+ def _parse_line(self, line, source_key, validcountries = None):
-+ # Skip version line
-+ if line.startswith("2"):
-+ return
-+
-+ # Skip comments
-+ if line.startswith("#"):
-+ return
-+
-+ try:
-+ registry, country_code, type, line = line.split("|", 3)
-+ except:
-+ log.warning("Could not parse line: %s" % line)
-+ return
-+
-+ # Skip any lines that are for stats only or do not have a country
-+ # code at all (avoids log spam below)
-+ if not country_code or country_code == '*':
-+ return
-+
-+ # Skip objects with unknown country codes
-+ if validcountries and country_code not in validcountries:
-+ log.warning("Skipping line with bogus country '%s': %s" % \
-+ (country_code, line))
-+ return
-+
-+ if type in ("ipv6", "ipv4"):
-+ return self._parse_ip_line(country_code, type, line, source_key)
-+
-+ def _parse_ip_line(self, country, type, line, source_key):
-+ try:
-+ address, prefix, date, status, organization = line.split("|")
-+ except ValueError:
-+ organization = None
-+
-+ # Try parsing the line without organization
-+ try:
-+ address, prefix, date, status = line.split("|")
-+ except ValueError:
-+ log.warning("Unhandled line format: %s" % line)
-+ return
-+
-+ # Skip anything that isn't properly assigned
-+ if not status in ("assigned", "allocated"):
-+ return
-+
-+ # Cast prefix into an integer
-+ try:
-+ prefix = int(prefix)
-+ except:
-+ log.warning("Invalid prefix: %s" % prefix)
-+ return
-+
-+ # Fix prefix length for IPv4
-+ if type == "ipv4":
-+ prefix = 32 - int(math.log(prefix, 2))
-+
-+ # Try to parse the address
-+ try:
-+ network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
-+ except ValueError:
-+ log.warning("Invalid IP address: %s" % address)
-+ return
-+
-+ if not self._check_parsed_network(network):
-+ return
-+
-+ self.db.execute("INSERT INTO networks(network, country, original_countries, source) \
-+ VALUES(%s, %s, %s, %s) ON CONFLICT (network) DO \
-+ UPDATE SET country = excluded.country",
-+ "%s" % network, country, [country], source_key,
-+ )
-+
-+ def _import_as_names_from_arin(self):
-+ downloader = location.importer.Downloader()
-+
-+ # XXX: Download AS names file from ARIN (note that these names appear to be quite
-+ # technical, not intended for human consumption, as description fields in
-+ # organisation handles for other RIRs are - however, this is what we have got,
-+ # and in some cases, it might be still better than nothing)
-+ with downloader.request("https://ftp.arin.net/info/asn.txt", return_blocks=False) as f:
-+ for line in f:
-+ # Convert binary line to string...
-+ line = str(line)
-+
-+ # ... valid lines start with a space, followed by the number of the Autonomous System ...
-+ if not line.startswith(" "):
-+ continue
-+
-+ # Split line and check if there is a valid ASN in it...
-+ asn, name = line.split()[0:2]
-+
-+ try:
-+ asn = int(asn)
-+ except ValueError:
-+ log.debug("Skipping ARIN AS names line not containing an integer for ASN")
-+ continue
-+
-+ # Filter invalid ASNs...
-+ if not self._check_parsed_asn(asn):
-+ continue
-+
-+ # Skip any AS name that appears to be a placeholder for a different RIR or entity...
-+ if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|DNIC|LACNIC|RIPE|IANA)(?:\d?$|\-)", name):
-+ continue
-+
-+ # Bail out in case the AS name contains anything we do not expect here...
-+ if re.search(r"[^a-zA-Z0-9-_]", name):
-+ log.debug("Skipping ARIN AS name for %s containing invalid characters: %s" % \
-+ (asn, name))
-+
-+ # Things look good here, run INSERT statement and skip this one if we already have
-+ # a (better?) name for this Autonomous System...
-+ self.db.execute("""
-+ INSERT INTO autnums(
-+ number,
-+ name,
-+ source
-+ ) VALUES (%s, %s, %s)
-+ ON CONFLICT (number) DO NOTHING""",
-+ asn,
-+ name,
-+ "ARIN",
-+ )
-+
-+ def handle_update_announcements(self, ns):
-+ server = ns.server[0]
-+
-+ with self.db.transaction():
-+ if server.startswith("/"):
-+ self._handle_update_announcements_from_bird(server)
-+ else:
-+ self._handle_update_announcements_from_telnet(server)
-+
-+ # Purge anything we never want here
-+ self.db.execute("""
-+ -- Delete default routes
-+ DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
-+
-+ -- Delete anything that is not global unicast address space
-+ DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
-+
-+ -- DELETE "current network" address space
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
-+
-+ -- DELETE local loopback address space
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
-+
-+ -- DELETE RFC 1918 address space
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
-+
-+ -- DELETE test, benchmark and documentation address space
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
-+
-+ -- DELETE CGNAT address space (RFC 6598)
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
-+
-+ -- DELETE link local address space
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
-+
-+ -- DELETE IPv6 to IPv4 (6to4) address space (RFC 3068)
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
-+ DELETE FROM announcements WHERE family(network) = 6 AND network <<= '2002::/16';
-+
-+ -- DELETE multicast and reserved address space
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
-+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
-+
-+ -- Delete networks that are too small to be in the global routing table
-+ DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
-+ DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
-+
-+ -- Delete any non-public or reserved ASNs
-+ DELETE FROM announcements WHERE NOT (
-+ (autnum >= 1 AND autnum <= 23455)
-+ OR
-+ (autnum >= 23457 AND autnum <= 64495)
-+ OR
-+ (autnum >= 131072 AND autnum <= 4199999999)
-+ );
-+
-+ -- Delete everything that we have not seen for 14 days
-+ DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
-+ """)
-+
-+ def _handle_update_announcements_from_bird(self, server):
-+ # Pre-compile the regular expression for faster searching
-+ route = re.compile(b"^\s(.+?)\s+.+?\[(?:AS(.*?))?.\]$")
-+
-+ log.info("Requesting routing table from Bird (%s)" % server)
-+
-+ aggregated_networks = []
-+
-+ # Send command to list all routes
-+ for line in self._bird_cmd(server, "show route"):
-+ m = route.match(line)
-+ if not m:
-+ # Skip empty lines
-+ if not line:
-+ pass
-+
-+ # Ignore any header lines with the name of the routing table
-+ elif line.startswith(b"Table"):
-+ pass
-+
-+ # Log anything else
-+ else:
-+ log.debug("Could not parse line: %s" % line.decode())
-+
-+ continue
-+
-+ # Fetch the extracted network and ASN
-+ network, autnum = m.groups()
-+
-+ # Decode into strings
-+ if network:
-+ network = network.decode()
-+ if autnum:
-+ autnum = autnum.decode()
-+
-+ # Collect all aggregated networks
-+ if not autnum:
-+ log.debug("%s is an aggregated network" % network)
-+ aggregated_networks.append(network)
-+ continue
-+
-+ # Insert it into the database
-+ self.db.execute("INSERT INTO announcements(network, autnum) \
-+ VALUES(%s, %s) ON CONFLICT (network) DO \
-+ UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
-+ network, autnum,
-+ )
-+
-+ # Process any aggregated networks
-+ for network in aggregated_networks:
-+ log.debug("Processing aggregated network %s" % network)
-+
-+ # Run "show route all" for each network
-+ for line in self._bird_cmd(server, "show route %s all" % network):
-+ # Try finding the path
-+ m = re.match(b"\s+BGP\.as_path:.* (\d+) {\d+}$", line)
-+ if m:
-+ # Select the last AS number in the path
-+ autnum = m.group(1).decode()
-+
-+ # Insert it into the database
-+ self.db.execute("INSERT INTO announcements(network, autnum) \
-+ VALUES(%s, %s) ON CONFLICT (network) DO \
-+ UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
-+ network, autnum,
-+ )
-+
-+ # We don't need to process any more
-+ break
-+
-+ def _handle_update_announcements_from_telnet(self, server):
-+ # Pre-compile regular expression for routes
-+ route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
-+
-+ with telnetlib.Telnet(server) as t:
-+ # Enable debug mode
-+ #if ns.debug:
-+ # t.set_debuglevel(10)
-+
-+ # Wait for console greeting
-+ greeting = t.read_until(b"> ", timeout=30)
-+ if not greeting:
-+ log.error("Could not get a console prompt")
-+ return 1
-+
-+ # Disable pagination
-+ t.write(b"terminal length 0\n")
-+
-+ # Wait for the prompt to return
-+ t.read_until(b"> ")
-+
-+ # Fetch the routing tables
-+ for protocol in ("ipv6", "ipv4"):
-+ log.info("Requesting %s routing table" % protocol)
-+
-+ # Request the full unicast routing table
-+ t.write(b"show bgp %s unicast\n" % protocol.encode())
-+
-+ # Read entire header which ends with "Path"
-+ t.read_until(b"Path\r\n")
-+
-+ while True:
-+ # Try reading a full entry
-+ # Those might be broken across multiple lines but ends with i
-+ line = t.read_until(b"i\r\n", timeout=5)
-+ if not line:
-+ break
-+
-+ # Show line for debugging
-+ #log.debug(repr(line))
-+
-+ # Try finding a route in here
-+ m = route.match(line)
-+ if m:
-+ network, autnum = m.groups()
-+
-+ # Convert network to string
-+ network = network.decode()
-+
-+ # Append /24 for IPv4 addresses
-+ if not "/" in network and not ":" in network:
-+ network = "%s/24" % network
-+
-+ # Convert AS number to integer
-+ autnum = int(autnum)
-+
-+ log.info("Found announcement for %s by %s" % (network, autnum))
-+
-+ self.db.execute("INSERT INTO announcements(network, autnum) \
-+ VALUES(%s, %s) ON CONFLICT (network) DO \
-+ UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
-+ network, autnum,
-+ )
-+
-+ log.info("Finished reading the %s routing table" % protocol)
-+
-+ def _bird_cmd(self, socket_path, command):
-+ # Connect to the socket
-+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-+ s.connect(socket_path)
-+
-+ # Allocate some buffer
-+ buffer = b""
-+
-+ log.debug("Sending Bird command: %s" % command)
-+
-+ # Send the command
-+ s.send(b"%s\n" % command.encode())
-+
-+ while True:
-+ # Fill up the buffer
-+ buffer += s.recv(4096)
-+
-+ while True:
-+ # Search for the next newline
-+ pos = buffer.find(b"\n")
-+
-+ # If we cannot find one, we go back and read more data
-+ if pos <= 0:
-+ break
-+
-+ # Cut after the newline character
-+ pos += 1
-+
-+ # Split the line we want and keep the rest in buffer
-+ line, buffer = buffer[:pos], buffer[pos:]
-+
-+ # Try parsing any status lines
-+ if len(line) > 4 and line[:4].isdigit() and line[4] in (32, 45):
-+ code, delim, line = int(line[:4]), line[4], line[5:]
-+
-+ log.debug("Received response code %s from bird" % code)
-+
-+ # End of output
-+ if code == 0:
-+ return
-+
-+ # Ignore hello line
-+ elif code == 1:
-+ continue
-+
-+ # Otherwise return the line
-+ yield line
-+
-+ def handle_update_overrides(self, ns):
-+ with self.db.transaction():
-+ # Drop all data that we have
-+ self.db.execute("""
-+ TRUNCATE TABLE autnum_overrides;
-+ TRUNCATE TABLE network_overrides;
-+ """)
-+
-+ # Update overrides for various cloud providers big enough to publish their own IP
-+ # network allocation lists in a machine-readable format...
-+ self._update_overrides_for_aws()
-+
-+ # Update overrides for Spamhaus DROP feeds...
-+ self._update_overrides_for_spamhaus_drop()
-+
-+ for file in ns.files:
-+ log.info("Reading %s..." % file)
-+
-+ with open(file, "rb") as f:
-+ for type, block in location.importer.read_blocks(f):
-+ if type == "net":
-+ network = block.get("net")
-+ # Try to parse and normalise the network
-+ try:
-+ network = ipaddress.ip_network(network, strict=False)
-+ except ValueError as e:
-+ log.warning("Invalid IP network: %s: %s" % (network, e))
-+ continue
-+
-+ # Prevent that we overwrite all networks
-+ if network.prefixlen == 0:
-+ log.warning("Skipping %s: You cannot overwrite default" % network)
-+ continue
-+
-+ self.db.execute("""
-+ INSERT INTO network_overrides(
-+ network,
-+ country,
-+ source,
-+ is_anonymous_proxy,
-+ is_satellite_provider,
-+ is_anycast,
-+ is_drop
-+ ) VALUES (%s, %s, %s, %s, %s, %s, %s)
-+ ON CONFLICT (network) DO NOTHING""",
-+ "%s" % network,
-+ block.get("country"),
-+ "manual",
-+ self._parse_bool(block, "is-anonymous-proxy"),
-+ self._parse_bool(block, "is-satellite-provider"),
-+ self._parse_bool(block, "is-anycast"),
-+ self._parse_bool(block, "drop"),
-+ )
-+
-+ elif type == "aut-num":
-+ autnum = block.get("aut-num")
-+
-+ # Check if AS number begins with "AS"
-+ if not autnum.startswith("AS"):
-+ log.warning("Invalid AS number: %s" % autnum)
-+ continue
-+
-+ # Strip "AS"
-+ autnum = autnum[2:]
-+
-+ self.db.execute("""
-+ INSERT INTO autnum_overrides(
-+ number,
-+ name,
-+ country,
-+ source,
-+ is_anonymous_proxy,
-+ is_satellite_provider,
-+ is_anycast,
-+ is_drop
-+ ) VALUES(%s, %s, %s, %s, %s, %s, %s, %s)
-+ ON CONFLICT DO NOTHING""",
-+ autnum,
-+ block.get("name"),
-+ block.get("country"),
-+ "manual",
-+ self._parse_bool(block, "is-anonymous-proxy"),
-+ self._parse_bool(block, "is-satellite-provider"),
-+ self._parse_bool(block, "is-anycast"),
-+ self._parse_bool(block, "drop"),
-+ )
-+
-+ else:
-+ log.warning("Unsupported type: %s" % type)
-+
-+ def _update_overrides_for_aws(self):
-+ # Download Amazon AWS IP allocation file to create overrides...
-+ downloader = location.importer.Downloader()
-+
-+ try:
-+ with downloader.request("https://ip-ranges.amazonaws.com/ip-ranges.json", return_blocks=False) as f:
-+ aws_ip_dump = json.load(f.body)
-+ except Exception as e:
-+ log.error("unable to preprocess Amazon AWS IP ranges: %s" % e)
-+ return
-+
-+ # XXX: Set up a dictionary for mapping a region name to a country. Unfortunately,
-+ # there seems to be no machine-readable version available of this other than
-+ # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html
-+ # (worse, it seems to be incomplete :-/ ); https://www.cloudping.cloud/endpoints
-+ # was helpful here as well.
-+ aws_region_country_map = {
-+ "af-south-1": "ZA",
-+ "ap-east-1": "HK",
-+ "ap-south-1": "IN",
-+ "ap-south-2": "IN",
-+ "ap-northeast-3": "JP",
-+ "ap-northeast-2": "KR",
-+ "ap-southeast-1": "SG",
-+ "ap-southeast-2": "AU",
-+ "ap-southeast-3": "MY",
-+ "ap-southeast-4": "AU",
-+ "ap-northeast-1": "JP",
-+ "ca-central-1": "CA",
-+ "eu-central-1": "DE",
-+ "eu-central-2": "CH",
-+ "eu-west-1": "IE",
-+ "eu-west-2": "GB",
-+ "eu-south-1": "IT",
-+ "eu-south-2": "ES",
-+ "eu-west-3": "FR",
-+ "eu-north-1": "SE",
-+ "il-central-1": "IL", # XXX: This one is not documented anywhere except for ip-ranges.json itself
-+ "me-central-1": "AE",
-+ "me-south-1": "BH",
-+ "sa-east-1": "BR"
-+ }
-+
-+ # Fetch all valid country codes to check parsed networks aganist...
-+ rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
-+ validcountries = []
-+
-+ for row in rows:
-+ validcountries.append(row.country_code)
-+
-+ with self.db.transaction():
-+ for snetwork in aws_ip_dump["prefixes"] + aws_ip_dump["ipv6_prefixes"]:
-+ try:
-+ network = ipaddress.ip_network(snetwork.get("ip_prefix") or snetwork.get("ipv6_prefix"), strict=False)
-+ except ValueError:
-+ log.warning("Unable to parse line: %s" % snetwork)
-+ continue
-+
-+ # Sanitize parsed networks...
-+ if not self._check_parsed_network(network):
-+ continue
-+
-+ # Determine region of this network...
-+ region = snetwork["region"]
-+ cc = None
-+ is_anycast = False
-+
-+ # Any region name starting with "us-" will get "US" country code assigned straight away...
-+ if region.startswith("us-"):
-+ cc = "US"
-+ elif region.startswith("cn-"):
-+ # ... same goes for China ...
-+ cc = "CN"
-+ elif region == "GLOBAL":
-+ # ... funny region name for anycast-like networks ...
-+ is_anycast = True
-+ elif region in aws_region_country_map:
-+ # ... assign looked up country code otherwise ...
-+ cc = aws_region_country_map[region]
-+ else:
-+ # ... and bail out if we are missing something here
-+ log.warning("Unable to determine country code for line: %s" % snetwork)
-+ continue
-+
-+ # Skip networks with unknown country codes
-+ if not is_anycast and validcountries and cc not in validcountries:
-+ log.warning("Skipping Amazon AWS network with bogus country '%s': %s" % \
-+ (cc, network))
-+ return
-+
-+ # Conduct SQL statement...
-+ self.db.execute("""
-+ INSERT INTO network_overrides(
-+ network,
-+ country,
-+ source,
-+ is_anonymous_proxy,
-+ is_satellite_provider,
-+ is_anycast
-+ ) VALUES (%s, %s, %s, %s, %s, %s)
-+ ON CONFLICT (network) DO NOTHING""",
-+ "%s" % network,
-+ cc,
-+ "Amazon AWS IP feed",
-+ None,
-+ None,
-+ is_anycast,
-+ )
-+
-+
-+ def _update_overrides_for_spamhaus_drop(self):
-+ downloader = location.importer.Downloader()
-+
-+ ip_urls = [
-+ "https://www.spamhaus.org/drop/drop.txt",
-+ "https://www.spamhaus.org/drop/edrop.txt",
-+ "https://www.spamhaus.org/drop/dropv6.txt"
-+ ]
-+
-+ asn_urls = [
-+ "https://www.spamhaus.org/drop/asndrop.txt"
-+ ]
-+
-+ for url in ip_urls:
-+ try:
-+ with downloader.request(url, return_blocks=False) as f:
-+ fcontent = f.body.readlines()
-+ except Exception as e:
-+ log.error("Unable to download Spamhaus DROP URL %s: %s" % (url, e))
-+ return
-+
-+ # Iterate through every line, filter comments and add remaining networks to
-+ # the override table in case they are valid...
-+ with self.db.transaction():
-+ for sline in fcontent:
-+
-+ # The response is assumed to be encoded in UTF-8...
-+ sline = sline.decode("utf-8")
-+
-+ # Comments start with a semicolon...
-+ if sline.startswith(";"):
-+ continue
-+
-+ # Extract network and ignore anything afterwards...
-+ try:
-+ network = ipaddress.ip_network(sline.split()[0], strict=False)
-+ except ValueError:
-+ log.error("Unable to parse line: %s" % sline)
-+ continue
-+
-+ # Sanitize parsed networks...
-+ if not self._check_parsed_network(network):
-+ log.warning("Skipping bogus network found in Spamhaus DROP URL %s: %s" % \
-+ (url, network))
-+ continue
-+
-+ # Conduct SQL statement...
-+ self.db.execute("""
-+ INSERT INTO network_overrides(
-+ network,
-+ source,
-+ is_drop
-+ ) VALUES (%s, %s, %s)
-+ ON CONFLICT (network) DO UPDATE SET is_drop = True""",
-+ "%s" % network,
-+ "Spamhaus DROP lists",
-+ True
-+ )
-+
-+ for url in asn_urls:
-+ try:
-+ with downloader.request(url, return_blocks=False) as f:
-+ fcontent = f.body.readlines()
-+ except Exception as e:
-+ log.error("Unable to download Spamhaus DROP URL %s: %s" % (url, e))
-+ return
-+
-+ # Iterate through every line, filter comments and add remaining ASNs to
-+ # the override table in case they are valid...
-+ with self.db.transaction():
-+ for sline in fcontent:
-+
-+ # The response is assumed to be encoded in UTF-8...
-+ sline = sline.decode("utf-8")
-+
-+ # Comments start with a semicolon...
-+ if sline.startswith(";"):
-+ continue
-+
-+ # Throw away anything after the first space...
-+ sline = sline.split()[0]
-+
-+ # ... strip the "AS" prefix from it ...
-+ sline = sline.strip("AS")
-+
-+ # ... and convert it into an integer. Voila.
-+ asn = int(sline)
-+
-+ # Filter invalid ASNs...
-+ if not self._check_parsed_asn(asn):
-+ log.warning("Skipping bogus ASN found in Spamhaus DROP URL %s: %s" % \
-+ (url, asn))
-+ continue
-+
-+ # Conduct SQL statement...
-+ self.db.execute("""
-+ INSERT INTO autnum_overrides(
-+ number,
-+ source,
-+ is_drop
-+ ) VALUES (%s, %s, %s)
-+ ON CONFLICT (number) DO UPDATE SET is_drop = True""",
-+ "%s" % asn,
-+ "Spamhaus ASN-DROP list",
-+ True
-+ )
-+
-+ @staticmethod
-+ def _parse_bool(block, key):
-+ val = block.get(key)
-+
-+ # There is no point to proceed when we got None
-+ if val is None:
-+ return
-+
-+ # Convert to lowercase
-+ val = val.lower()
-+
-+ # True
-+ if val in ("yes", "1"):
-+ return True
-+
-+ # False
-+ if val in ("no", "0"):
-+ return False
-+
-+ # Default to None
-+ return None
-+
-+ def handle_import_countries(self, ns):
-+ with self.db.transaction():
-+ # Drop all data that we have
-+ self.db.execute("TRUNCATE TABLE countries")
-+
-+ for file in ns.file:
-+ for line in file:
-+ line = line.rstrip()
-+
-+ # Ignore any comments
-+ if line.startswith("#"):
-+ continue
-+
-+ try:
-+ country_code, continent_code, name = line.split(maxsplit=2)
-+ except:
-+ log.warning("Could not parse line: %s" % line)
-+ continue
-+
-+ self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
-+ VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
-+
-+
-+def split_line(line):
-+ key, colon, val = line.partition(":")
-+
-+ # Strip any excess space
-+ key = key.strip()
-+ val = val.strip()
-+
-+ return key, val
-+
-+def main():
-+ # Run the command line interface
-+ c = CLI()
-+ c.run()
-+
-+main()
-diff --git a/src/scripts/location.in b/src/scripts/location.in
-new file mode 100644
-index 0000000..233cea0
---- /dev/null
-+++ b/src/scripts/location.in
-@@ -0,0 +1,644 @@
-+#!/usr/bin/python3
-+###############################################################################
-+# #
-+# libloc - A library to determine the location of someone on the Internet #
-+# #
-+# Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org> #
-+# #
-+# This library is free software; you can redistribute it and/or #
-+# modify it under the terms of the GNU Lesser General Public #
-+# License as published by the Free Software Foundation; either #
-+# version 2.1 of the License, or (at your option) any later version. #
-+# #
-+# This library is distributed in the hope that it will be useful, #
-+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
-+# Lesser General Public License for more details. #
-+# #
-+###############################################################################
-+
-+import argparse
-+import datetime
-+import ipaddress
-+import logging
-+import os
-+import re
-+import shutil
-+import socket
-+import sys
-+import time
-+
-+# Load our location module
-+import location
-+import location.downloader
-+import location.export
-+
-+from location.i18n import _
-+
-+# Setup logging
-+log = logging.getLogger("location")
-+
-+# Output formatters
-+
-+class CLI(object):
-+ def parse_cli(self):
-+ parser = argparse.ArgumentParser(
-+ description=_("Location Database Command Line Interface"),
-+ )
-+ subparsers = parser.add_subparsers()
-+
-+ # Global configuration flags
-+ parser.add_argument("--debug", action="store_true",
-+ help=_("Enable debug output"))
-+ parser.add_argument("--quiet", action="store_true",
-+ help=_("Enable quiet mode"))
-+
-+ # version
-+ parser.add_argument("--version", action="version",
-+ version="%(prog)s @VERSION@")
-+
-+ # database
-+ parser.add_argument("--database", "-d",
-+ default="@databasedir@/database.db", help=_("Path to database"),
-+ )
-+
-+ # public key
-+ parser.add_argument("--public-key", "-k",
-+ default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
-+ )
-+
-+ # Show the database version
-+ version = subparsers.add_parser("version",
-+ help=_("Show database version"))
-+ version.set_defaults(func=self.handle_version)
-+
-+ # lookup an IP address
-+ lookup = subparsers.add_parser("lookup",
-+ help=_("Lookup one or multiple IP addresses"),
-+ )
-+ lookup.add_argument("address", nargs="+")
-+ lookup.set_defaults(func=self.handle_lookup)
-+
-+ # Dump the whole database
-+ dump = subparsers.add_parser("dump",
-+ help=_("Dump the entire database"),
-+ )
-+ dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
-+ dump.set_defaults(func=self.handle_dump)
-+
-+ # Update
-+ update = subparsers.add_parser("update", help=_("Update database"))
-+ update.add_argument("--cron",
-+ help=_("Update the library only once per interval"),
-+ choices=("daily", "weekly", "monthly"),
-+ )
-+ update.set_defaults(func=self.handle_update)
-+
-+ # Verify
-+ verify = subparsers.add_parser("verify",
-+ help=_("Verify the downloaded database"))
-+ verify.set_defaults(func=self.handle_verify)
-+
-+ # Get AS
-+ get_as = subparsers.add_parser("get-as",
-+ help=_("Get information about one or multiple Autonomous Systems"),
-+ )
-+ get_as.add_argument("asn", nargs="+")
-+ get_as.set_defaults(func=self.handle_get_as)
-+
-+ # Search for AS
-+ search_as = subparsers.add_parser("search-as",
-+ help=_("Search for Autonomous Systems that match the string"),
-+ )
-+ search_as.add_argument("query", nargs=1)
-+ search_as.set_defaults(func=self.handle_search_as)
-+
-+ # List all networks in an AS
-+ list_networks_by_as = subparsers.add_parser("list-networks-by-as",
-+ help=_("Lists all networks in an AS"),
-+ )
-+ list_networks_by_as.add_argument("asn", nargs=1, type=int)
-+ list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
-+ list_networks_by_as.add_argument("--format",
-+ choices=location.export.formats.keys(), default="list")
-+ list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
-+
-+ # List all networks in a country
-+ list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
-+ help=_("Lists all networks in a country"),
-+ )
-+ list_networks_by_cc.add_argument("country_code", nargs=1)
-+ list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
-+ list_networks_by_cc.add_argument("--format",
-+ choices=location.export.formats.keys(), default="list")
-+ list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
-+
-+ # List all networks with flags
-+ list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
-+ help=_("Lists all networks with flags"),
-+ )
-+ list_networks_by_flags.add_argument("--anonymous-proxy",
-+ action="store_true", help=_("Anonymous Proxies"),
-+ )
-+ list_networks_by_flags.add_argument("--satellite-provider",
-+ action="store_true", help=_("Satellite Providers"),
-+ )
-+ list_networks_by_flags.add_argument("--anycast",
-+ action="store_true", help=_("Anycasts"),
-+ )
-+ list_networks_by_flags.add_argument("--drop",
-+ action="store_true", help=_("Hostile Networks safe to drop"),
-+ )
-+ list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
-+ list_networks_by_flags.add_argument("--format",
-+ choices=location.export.formats.keys(), default="list")
-+ list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
-+
-+ # List bogons
-+ list_bogons = subparsers.add_parser("list-bogons",
-+ help=_("Lists all bogons"),
-+ )
-+ list_bogons.add_argument("--family", choices=("ipv6", "ipv4"))
-+ list_bogons.add_argument("--format",
-+ choices=location.export.formats.keys(), default="list")
-+ list_bogons.set_defaults(func=self.handle_list_bogons)
-+
-+ # List countries
-+ list_countries = subparsers.add_parser("list-countries",
-+ help=_("Lists all countries"),
-+ )
-+ list_countries.add_argument("--show-name",
-+ action="store_true", help=_("Show the name of the country"),
-+ )
-+ list_countries.add_argument("--show-continent",
-+ action="store_true", help=_("Show the continent"),
-+ )
-+ list_countries.set_defaults(func=self.handle_list_countries)
-+
-+ # Export
-+ export = subparsers.add_parser("export",
-+ help=_("Exports data in many formats to load it into packet filters"),
-+ )
-+ export.add_argument("--format", help=_("Output format"),
-+ choices=location.export.formats.keys(), default="list")
-+ export.add_argument("--directory", help=_("Output directory"))
-+ export.add_argument("--family",
-+ help=_("Specify address family"), choices=("ipv6", "ipv4"),
-+ )
-+ export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
-+ export.set_defaults(func=self.handle_export)
-+
-+ args = parser.parse_args()
-+
-+ # Configure logging
-+ if args.debug:
-+ location.logger.set_level(logging.DEBUG)
-+ elif args.quiet:
-+ location.logger.set_level(logging.WARNING)
-+
-+ # Print usage if no action was given
-+ if not "func" in args:
-+ parser.print_usage()
-+ sys.exit(2)
-+
-+ return args
-+
-+ def run(self):
-+ # Parse command line arguments
-+ args = self.parse_cli()
-+
-+ # Open database
-+ try:
-+ db = location.Database(args.database)
-+ except FileNotFoundError as e:
-+ # Allow continuing without a database
-+ if args.func == self.handle_update:
-+ db = None
-+
-+ else:
-+ sys.stderr.write("location: Could not open database %s: %s\n" \
-+ % (args.database, e))
-+ sys.exit(1)
-+
-+ # Translate family (if present)
-+ if "family" in args:
-+ if args.family == "ipv6":
-+ args.family = socket.AF_INET6
-+ elif args.family == "ipv4":
-+ args.family = socket.AF_INET
-+ else:
-+ args.family = 0
-+
-+ # Call function
-+ try:
-+ ret = args.func(db, args)
-+
-+ # Catch invalid inputs
-+ except ValueError as e:
-+ sys.stderr.write("%s\n" % e)
-+ ret = 2
-+
-+ # Catch any other exceptions
-+ except Exception as e:
-+ sys.stderr.write("%s\n" % e)
-+ ret = 1
-+
-+ # Return with exit code
-+ if ret:
-+ sys.exit(ret)
-+
-+ # Otherwise just exit
-+ sys.exit(0)
-+
-+ def handle_version(self, db, ns):
-+ """
-+ Print the version of the database
-+ """
-+ t = time.strftime(
-+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
-+ )
-+
-+ print(t)
-+
-+ def handle_lookup(self, db, ns):
-+ ret = 0
-+
-+ format = " %-24s: %s"
-+
-+ for address in ns.address:
-+ try:
-+ network = db.lookup(address)
-+ except ValueError:
-+ print(_("Invalid IP address: %s") % address, file=sys.stderr)
-+ return 2
-+
-+ args = {
-+ "address" : address,
-+ "network" : network,
-+ }
-+
-+ # Nothing found?
-+ if not network:
-+ print(_("Nothing found for %(address)s") % args, file=sys.stderr)
-+ ret = 1
-+ continue
-+
-+ print("%s:" % address)
-+ print(format % (_("Network"), network))
-+
-+ # Print country
-+ if network.country_code:
-+ country = db.get_country(network.country_code)
-+
-+ print(format % (
-+ _("Country"),
-+ country.name if country else network.country_code),
-+ )
-+
-+ # Print AS information
-+ if network.asn:
-+ autonomous_system = db.get_as(network.asn)
-+
-+ print(format % (
-+ _("Autonomous System"),
-+ autonomous_system or "AS%s" % network.asn),
-+ )
-+
-+ # Anonymous Proxy
-+ if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
-+ print(format % (
-+ _("Anonymous Proxy"), _("yes"),
-+ ))
-+
-+ # Satellite Provider
-+ if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
-+ print(format % (
-+ _("Satellite Provider"), _("yes"),
-+ ))
-+
-+ # Anycast
-+ if network.has_flag(location.NETWORK_FLAG_ANYCAST):
-+ print(format % (
-+ _("Anycast"), _("yes"),
-+ ))
-+
-+ # Hostile Network
-+ if network.has_flag(location.NETWORK_FLAG_DROP):
-+ print(format % (
-+ _("Hostile Network safe to drop"), _("yes"),
-+ ))
-+
-+ return ret
-+
-+ def handle_dump(self, db, ns):
-+ # Use output file or write to stdout
-+ f = ns.output or sys.stdout
-+
-+ # Format everything like this
-+ format = "%-24s %s\n"
-+
-+ # Write metadata
-+ f.write("#\n# Location Database Export\n#\n")
-+
-+ f.write("# Generated: %s\n" % time.strftime(
-+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
-+ ))
-+
-+ if db.vendor:
-+ f.write("# Vendor: %s\n" % db.vendor)
-+
-+ if db.license:
-+ f.write("# License: %s\n" % db.license)
-+
-+ f.write("#\n")
-+
-+ if db.description:
-+ for line in db.description.splitlines():
-+ line = "# %s" % line
-+ f.write("%s\n" % line.rstrip())
-+
-+ f.write("#\n")
-+
-+ # Iterate over all ASes
-+ for a in db.ases:
-+ f.write("\n")
-+ f.write(format % ("aut-num:", "AS%s" % a.number))
-+ f.write(format % ("name:", a.name))
-+
-+ flags = {
-+ location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:",
-+ location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
-+ location.NETWORK_FLAG_ANYCAST : "is-anycast:",
-+ location.NETWORK_FLAG_DROP : "drop:",
-+ }
-+
-+ # Iterate over all networks
-+ for n in db.networks:
-+ f.write("\n")
-+ f.write(format % ("net:", n))
-+
-+ if n.country_code:
-+ f.write(format % ("country:", n.country_code))
-+
-+ if n.asn:
-+ f.write(format % ("aut-num:", n.asn))
-+
-+ # Print all flags
-+ for flag in flags:
-+ if n.has_flag(flag):
-+ f.write(format % (flags[flag], "yes"))
-+
-+ def handle_get_as(self, db, ns):
-+ """
-+ Gets information about Autonomous Systems
-+ """
-+ ret = 0
-+
-+ for asn in ns.asn:
-+ try:
-+ asn = int(asn)
-+ except ValueError:
-+ print(_("Invalid ASN: %s") % asn, file=sys.stderr)
-+ ret = 1
-+ continue
-+
-+ # Fetch AS from database
-+ a = db.get_as(asn)
-+
-+ # Nothing found
-+ if not a:
-+ print(_("Could not find AS%s") % asn, file=sys.stderr)
-+ ret = 1
-+ continue
-+
-+ print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
-+
-+ return ret
-+
-+ def handle_search_as(self, db, ns):
-+ for query in ns.query:
-+ # Print all matches ASes
-+ for a in db.search_as(query):
-+ print(a)
-+
-+ def handle_update(self, db, ns):
-+ if ns.cron and db:
-+ now = time.time()
-+
-+ if ns.cron == "daily":
-+ delta = datetime.timedelta(days=1)
-+ elif ns.cron == "weekly":
-+ delta = datetime.timedelta(days=7)
-+ elif ns.cron == "monthly":
-+ delta = datetime.timedelta(days=30)
-+
-+ delta = delta.total_seconds()
-+
-+ # Check if the database has recently been updated
-+ if db.created_at >= (now - delta):
-+ log.info(
-+ _("The database has been updated recently"),
-+ )
-+ return 3
-+
-+ # Fetch the timestamp we need from DNS
-+ t = location.discover_latest_version()
-+
-+ # Check the version of the local database
-+ if db and t and db.created_at >= t:
-+ log.info("Already on the latest version")
-+ return
-+
-+ # Download the database into the correct directory
-+ tmpdir = os.path.dirname(ns.database)
-+
-+ # Create a downloader
-+ d = location.downloader.Downloader()
-+
-+ # Try downloading a new database
-+ try:
-+ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
-+
-+ # If no file could be downloaded, log a message
-+ except FileNotFoundError as e:
-+ log.error("Could not download a new database")
-+ return 1
-+
-+ # If we have not received a new file, there is nothing to do
-+ if not t:
-+ return 3
-+
-+ # Move temporary file to destination
-+ shutil.move(t.name, ns.database)
-+
-+ return 0
-+
-+ def handle_verify(self, db, ns):
-+ # Verify the database
-+ with open(ns.public_key, "r") as f:
-+ if not db.verify(f):
-+ log.error("Could not verify database")
-+ return 1
-+
-+ # Success
-+ log.debug("Database successfully verified")
-+ return 0
-+
-+ def __get_output_formatter(self, ns):
-+ try:
-+ cls = location.export.formats[ns.format]
-+ except KeyError:
-+ cls = location.export.OutputFormatter
-+
-+ return cls
-+
-+ def handle_list_countries(self, db, ns):
-+ for country in db.countries:
-+ line = [
-+ country.code,
-+ ]
-+
-+ if ns.show_continent:
-+ line.append(country.continent_code)
-+
-+ if ns.show_name:
-+ line.append(country.name)
-+
-+ # Format the output
-+ line = " ".join(line)
-+
-+ # Print the output
-+ print(line)
-+
-+ def handle_list_networks_by_as(self, db, ns):
-+ writer = self.__get_output_formatter(ns)
-+
-+ for asn in ns.asn:
-+ f = writer("AS%s" % asn, f=sys.stdout)
-+
-+ # Print all matching networks
-+ for n in db.search_networks(asns=[asn], family=ns.family):
-+ f.write(n)
-+
-+ f.finish()
-+
-+ def handle_list_networks_by_cc(self, db, ns):
-+ writer = self.__get_output_formatter(ns)
-+
-+ for country_code in ns.country_code:
-+ # Open standard output
-+ f = writer(country_code, f=sys.stdout)
-+
-+ # Print all matching networks
-+ for n in db.search_networks(country_codes=[country_code], family=ns.family):
-+ f.write(n)
-+
-+ f.finish()
-+
-+ def handle_list_networks_by_flags(self, db, ns):
-+ flags = 0
-+
-+ if ns.anonymous_proxy:
-+ flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
-+
-+ if ns.satellite_provider:
-+ flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
-+
-+ if ns.anycast:
-+ flags |= location.NETWORK_FLAG_ANYCAST
-+
-+ if ns.drop:
-+ flags |= location.NETWORK_FLAG_DROP
-+
-+ if not flags:
-+ raise ValueError(_("You must at least pass one flag"))
-+
-+ writer = self.__get_output_formatter(ns)
-+ f = writer("custom", f=sys.stdout)
-+
-+ for n in db.search_networks(flags=flags, family=ns.family):
-+ f.write(n)
-+
-+ f.finish()
-+
-+ def handle_list_bogons(self, db, ns):
-+ writer = self.__get_output_formatter(ns)
-+ f = writer("bogons", f=sys.stdout)
-+
-+ for n in db.list_bogons(family=ns.family):
-+ f.write(n)
-+
-+ f.finish()
-+
-+ def handle_export(self, db, ns):
-+ countries, asns = [], []
-+
-+ # Translate family
-+ if ns.family:
-+ families = [ ns.family ]
-+ else:
-+ families = [ socket.AF_INET6, socket.AF_INET ]
-+
-+ for object in ns.objects:
-+ m = re.match("^AS(\d+)$", object)
-+ if m:
-+ object = int(m.group(1))
-+
-+ asns.append(object)
-+
-+ elif location.country_code_is_valid(object) \
-+ or object in ("A1", "A2", "A3", "XD"):
-+ countries.append(object)
-+
-+ else:
-+ log.warning("Invalid argument: %s" % object)
-+ continue
-+
-+ # Default to exporting all countries
-+ if not countries and not asns:
-+ countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries]
-+
-+ # Select the output format
-+ writer = self.__get_output_formatter(ns)
-+
-+ e = location.export.Exporter(db, writer)
-+ e.export(ns.directory, countries=countries, asns=asns, families=families)
-+
-+
-+def format_timedelta(t):
-+ s = []
-+
-+ if t.days:
-+ s.append(
-+ _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
-+ )
-+
-+ hours = t.seconds // 3600
-+ if hours:
-+ s.append(
-+ _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
-+ )
-+
-+ minutes = (t.seconds % 3600) // 60
-+ if minutes:
-+ s.append(
-+ _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
-+ )
-+
-+ seconds = t.seconds % 60
-+ if t.seconds:
-+ s.append(
-+ _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
-+ )
-+
-+ if not s:
-+ return _("Now")
-+
-+ return _("%s ago") % ", ".join(s)
-+
-+def main():
-+ # Run the command line interface
-+ c = CLI()
-+ c.run()
-+
-+main()
projects / location / debian / libloc.git / commitdiff
