[location/libloc.git] / src / python / location-importer.in

#!/usr/bin/python3
###############################################################################
#                                                                             #
# libloc - A library to determine the location of someone on the Internet     #
#                                                                             #
# Copyright (C) 2020 IPFire Development Team <info@ipfire.org>                #
#                                                                             #
# This library is free software; you can redistribute it and/or               #
# modify it under the terms of the GNU Lesser General Public                  #
# License as published by the Free Software Foundation; either                #
# version 2.1 of the License, or (at your option) any later version.          #
#                                                                             #
# This library is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
# Lesser General Public License for more details.                             #
#                                                                             #
###############################################################################

import argparse
import ipaddress
import logging
import math
import re
import sys

# Load our location module
import location
import location.database
import location.importer
from location.i18n import _

# Initialise logging
log = logging.getLogger("location.importer")
log.propagate = 1

INVALID_ADDRESSES = (
	"0.0.0.0",
	"::/0",
	"0::/0",
)

class CLI(object):
	def parse_cli(self):
		parser = argparse.ArgumentParser(
			description=_("Location Importer Command Line Interface"),
		)
		subparsers = parser.add_subparsers()

		# Global configuration flags
		parser.add_argument("--debug", action="store_true",
			help=_("Enable debug output"))

		# version
		parser.add_argument("--version", action="version",
			version="%(prog)s @VERSION@")

		# Database
		parser.add_argument("--database-host", required=True,
			help=_("Database Hostname"), metavar=_("HOST"))
		parser.add_argument("--database-name", required=True,
			help=_("Database Name"), metavar=_("NAME"))
		parser.add_argument("--database-username", required=True,
			help=_("Database Username"), metavar=_("USERNAME"))
		parser.add_argument("--database-password", required=True,
			help=_("Database Password"), metavar=_("PASSWORD"))

		# Update WHOIS
		update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
		update_whois.set_defaults(func=self.handle_update_whois)

		args = parser.parse_args()

		# Enable debug logging
		if args.debug:
			log.setLevel(logging.DEBUG)

		# Print usage if no action was given
		if not "func" in args:
			parser.print_usage()
			sys.exit(2)

		return args

	def run(self):
		# Parse command line arguments
		args = self.parse_cli()

		# Initialise database
		self.db = self._setup_database(args)

		# Call function
		ret = args.func(args)

		# Return with exit code
		if ret:
			sys.exit(ret)

		# Otherwise just exit
		sys.exit(0)

	def _setup_database(self, ns):
		"""
			Initialise the database
		"""
		# Connect to database
		db = location.database.Connection(
			host=ns.database_host, database=ns.database_name,
			user=ns.database_username, password=ns.database_password,
		)

		with db.transaction():
			db.execute("""
				-- autnums
				CREATE TABLE IF NOT EXISTS autnums(number integer, name text, organization text);
				CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);

				-- inetnums
				CREATE TABLE IF NOT EXISTS inetnums(network inet, name text, country text, description text);
				CREATE UNIQUE INDEX IF NOT EXISTS inetnums_networks ON inetnums(network);
				CREATE INDEX IF NOT EXISTS inetnums_family ON inetnums(family(network));

				-- organizations
				CREATE TABLE IF NOT EXISTS organizations(handle text, name text, country text);
				CREATE UNIQUE INDEX IF NOT EXISTS organizations_handle ON organizations(handle);

				-- routes
				CREATE TABLE IF NOT EXISTS routes(network inet, asn integer);
				CREATE UNIQUE INDEX IF NOT EXISTS routes_network ON routes(network);
				CREATE INDEX IF NOT EXISTS routes_family ON routes(family(network));
			""")

		return db

	def handle_update_whois(self, ns):
		downloader = location.importer.Downloader()

		# Download all sources
		for source in location.importer.WHOIS_SOURCES:
			with self.db.transaction():
				with downloader.request(source, return_blocks=True) as f:
					for block in f:
						self._parse_block(block)

	def _parse_block(self, block):
		# Get first line to find out what type of block this is
		line = block[0]

		# inetnum
		if line.startswith("inet6num:") or line.startswith("inetnum:"):
			return self._parse_inetnum_block(block)

		# route
		elif line.startswith("route6:") or line.startswith("route:"):
			return self._parse_route_block(block)

		# aut-num
		elif line.startswith("aut-num:"):
			return self._parse_autnum_block(block)

		# organisation
		elif line.startswith("organisation:"):
			return self._parse_org_block(block)

		# person (ignored)
		elif line.startswith("person:"):
			return

		# domain (ignored)
		elif line.startswith("domain:"):
			return

		# mntner (ignored)
		elif line.startswith("mntner:"):
			return

		# as-block (ignored)
		elif line.startswith("as-block:"):
			return

		# as-set (ignored)
		elif line.startswith("as-set:"):
			return

		# route-set (ignored)
		elif line.startswith("route-set:"):
			return

		# role (ignored)
		elif line.startswith("role:"):
			return

		# key-cert (ignored)
		elif line.startswith("key-cert:"):
			return

		# irt (ignored)
		elif line.startswith("irt:"):
			return

		# Log any unknown blocks
		else:
			log.warning("Unknown block:")
			for line in block:
				log.warning(line)

	def _parse_autnum_block(self, block):
		log.debug("Parsing autnum block:")

		autnum = {}
		for line in block:
			# Split line
			key, val = split_line(line)

			if key == "aut-num":
				m = re.match(r"^(AS|as)(\d+)", val)
				if m:
					autnum["asn"] = m.group(2)

			elif key in ("as-name", "org"):
				autnum[key] = val

		# Skip empty objects
		if not autnum:
			return

		# Insert into database
		self.db.execute("INSERT INTO autnums(number, name, organization) \
			VALUES(%s, %s, %s) ON CONFLICT (number) DO UPDATE SET \
				name = excluded.name, organization = excluded.organization",
			autnum.get("asn"), autnum.get("as-name"), autnum.get("org"),
		)

	def _parse_inetnum_block(self, block):
		inetnum = {}
		for line in block:
			# Split line
			key, val = split_line(line)

			if key == "inetnum":
				start_address, delim, end_address = val.partition("-")

				# Strip any excess space
				start_address, end_address = start_address.rstrip(), end_address.strip()

				# Skip invalid blocks
				if start_address in INVALID_ADDRESSES:
					return

				# Convert to IP address
				try:
					start_address = ipaddress.ip_address(start_address)
					end_address   = ipaddress.ip_address(end_address)
				except ValueError:
					log.warning("Could not parse line: %s" % line)
					return

				# Set prefix to default
				prefix = 32

				# Count number of addresses in this subnet
				num_addresses = int(end_address) - int(start_address)
				if num_addresses:
					prefix -= math.log(num_addresses, 2)

				inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)

			elif key == "inet6num":
				# Skip invalid blocks
				if val in INVALID_ADDRESSES:
					return

				inetnum[key] = val

			elif key == "netname":
				inetnum[key] = val

			elif key == "country":
				if val == "UNITED STATES":
					val = "US"

				inetnum[key] = val.upper()

			elif key == "descr":
				if key in inetnum:
					inetnum[key] += "\n%s" % val
				else:
					inetnum[key] = val

		# Skip empty objects
		if not inetnum:
			return

		network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)

		self.db.execute("INSERT INTO inetnums(network, name, country, description) \
			VALUES(%s, %s, %s, %s) ON CONFLICT (network) DO \
			UPDATE SET name = excluded.name, country = excluded.country, description = excluded.description",
			"%s" % network, inetnum.get("netname"), inetnum.get("country"), inetnum.get("descr"),
		)

	def _parse_org_block(self, block):
		org = {}
		for line in block:
			# Split line
			key, val = split_line(line)

			if key in ("organisation", "org-name", "country"):
				org[key] = val

		# Skip empty objects
		if not org:
			return

		self.db.execute("INSERT INTO organizations(handle, name, country) \
			VALUES(%s, %s, %s) ON CONFLICT (handle) DO \
			UPDATE SET name = excluded.name, country = excluded.country",
			org.get("organisation"), org.get("org-name"), org.get("country"),
		)

	def _parse_route_block(self, block):
		route = {}
		for line in block:
			# Split line
			key, val = split_line(line)

			# Keep any significant data
			if key in ("route6", "route"):
				route[key] = val

			elif key == "origin":
				m = re.match(r"^(AS|as)(\d+)", val)
				if m:
					route["asn"] = m.group(2)

		# Skip empty objects
		if not route:
			return

		network = ipaddress.ip_network(route.get("route6") or route.get("route"), strict=False)

		self.db.execute("INSERT INTO routes(network, asn) \
			VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET asn = excluded.asn",
			"%s" % network, route.get("asn"),
		)


def split_line(line):
	key, colon, val = line.partition(":")

	# Strip any excess space
	key = key.strip()
	val = val.strip()

	return key, val

def main():
	# Run the command line interface
	c = CLI()
	c.run()

main()
Commit	Line	Data
78ff0cf2 MT	1	#!/usr/bin/python3
	2	###############################################################################
	3	# #
	4	# libloc - A library to determine the location of someone on the Internet #
	5	# #
	6	# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
	7	# #
	8	# This library is free software; you can redistribute it and/or #
	9	# modify it under the terms of the GNU Lesser General Public #
	10	# License as published by the Free Software Foundation; either #
	11	# version 2.1 of the License, or (at your option) any later version. #
	12	# #
	13	# This library is distributed in the hope that it will be useful, #
	14	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	15	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
	16	# Lesser General Public License for more details. #
	17	# #
	18	###############################################################################
	19
	20	import argparse
6ffd06b5	21	import ipaddress
78ff0cf2	22	import logging
6ffd06b5 MT	23	import math
6ffd06b5 MT	24	import re
78ff0cf2 MT	25	import sys
	26
	27	# Load our location module
	28	import location
29c6fa22	29	import location.database
3192b66c	30	import location.importer
78ff0cf2 MT	31	from location.i18n import _
	32
	33	# Initialise logging
	34	log = logging.getLogger("location.importer")
	35	log.propagate = 1
	36
6ffd06b5 MT	37	INVALID_ADDRESSES = (
	38	"0.0.0.0",
	39	"::/0",
	40	"0::/0",
	41	)
	42
78ff0cf2 MT	43	class CLI(object):
	44	def parse_cli(self):
	45	parser = argparse.ArgumentParser(
	46	description=_("Location Importer Command Line Interface"),
	47	)
6ffd06b5	48	subparsers = parser.add_subparsers()
78ff0cf2 MT	49
	50	# Global configuration flags
	51	parser.add_argument("--debug", action="store_true",
	52	help=_("Enable debug output"))
	53
	54	# version
	55	parser.add_argument("--version", action="version",
	56	version="%(prog)s @VERSION@")
	57
29c6fa22 MT	58	# Database
	59	parser.add_argument("--database-host", required=True,
	60	help=_("Database Hostname"), metavar=_("HOST"))
	61	parser.add_argument("--database-name", required=True,
	62	help=_("Database Name"), metavar=_("NAME"))
	63	parser.add_argument("--database-username", required=True,
	64	help=_("Database Username"), metavar=_("USERNAME"))
	65	parser.add_argument("--database-password", required=True,
	66	help=_("Database Password"), metavar=_("PASSWORD"))
	67
6ffd06b5 MT	68	# Update WHOIS
	69	update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
	70	update_whois.set_defaults(func=self.handle_update_whois)
	71
78ff0cf2 MT	72	args = parser.parse_args()
	73
	74	# Enable debug logging
	75	if args.debug:
	76	log.setLevel(logging.DEBUG)
	77
6ffd06b5 MT	78	# Print usage if no action was given
	79	if not "func" in args:
	80	parser.print_usage()
	81	sys.exit(2)
	82
78ff0cf2 MT	83	return args
	84
	85	def run(self):
	86	# Parse command line arguments
	87	args = self.parse_cli()
	88
29c6fa22	89	# Initialise database
6ffd06b5	90	self.db = self._setup_database(args)
29c6fa22	91
78ff0cf2	92	# Call function
6ffd06b5	93	ret = args.func(args)
78ff0cf2 MT	94
	95	# Return with exit code
	96	if ret:
	97	sys.exit(ret)
	98
	99	# Otherwise just exit
	100	sys.exit(0)
	101
29c6fa22 MT	102	def _setup_database(self, ns):
	103	"""
	104	Initialise the database
	105	"""
	106	# Connect to database
	107	db = location.database.Connection(
	108	host=ns.database_host, database=ns.database_name,
	109	user=ns.database_username, password=ns.database_password,
	110	)
	111
	112	with db.transaction():
	113	db.execute("""
6ffd06b5 MT	114	-- autnums
	115	CREATE TABLE IF NOT EXISTS autnums(number integer, name text, organization text);
	116	CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
	117
	118	-- inetnums
	119	CREATE TABLE IF NOT EXISTS inetnums(network inet, name text, country text, description text);
	120	CREATE UNIQUE INDEX IF NOT EXISTS inetnums_networks ON inetnums(network);
	121	CREATE INDEX IF NOT EXISTS inetnums_family ON inetnums(family(network));
	122
	123	-- organizations
	124	CREATE TABLE IF NOT EXISTS organizations(handle text, name text, country text);
	125	CREATE UNIQUE INDEX IF NOT EXISTS organizations_handle ON organizations(handle);
	126
	127	-- routes
	128	CREATE TABLE IF NOT EXISTS routes(network inet, asn integer);
	129	CREATE UNIQUE INDEX IF NOT EXISTS routes_network ON routes(network);
	130	CREATE INDEX IF NOT EXISTS routes_family ON routes(family(network));
29c6fa22 MT	131	""")
	132
	133	return db
	134
6ffd06b5 MT	135	def handle_update_whois(self, ns):
	136	downloader = location.importer.Downloader()
	137
	138	# Download all sources
	139	for source in location.importer.WHOIS_SOURCES:
	140	with self.db.transaction():
	141	with downloader.request(source, return_blocks=True) as f:
	142	for block in f:
	143	self._parse_block(block)
	144
	145	def _parse_block(self, block):
	146	# Get first line to find out what type of block this is
	147	line = block[0]
	148
	149	# inetnum
	150	if line.startswith("inet6num:") or line.startswith("inetnum:"):
	151	return self._parse_inetnum_block(block)
	152
	153	# route
	154	elif line.startswith("route6:") or line.startswith("route:"):
	155	return self._parse_route_block(block)
	156
	157	# aut-num
	158	elif line.startswith("aut-num:"):
	159	return self._parse_autnum_block(block)
	160
	161	# organisation
	162	elif line.startswith("organisation:"):
	163	return self._parse_org_block(block)
	164
	165	# person (ignored)
	166	elif line.startswith("person:"):
	167	return
	168
	169	# domain (ignored)
	170	elif line.startswith("domain:"):
	171	return
	172
	173	# mntner (ignored)
	174	elif line.startswith("mntner:"):
	175	return
	176
	177	# as-block (ignored)
	178	elif line.startswith("as-block:"):
	179	return
	180
	181	# as-set (ignored)
	182	elif line.startswith("as-set:"):
	183	return
	184
	185	# route-set (ignored)
	186	elif line.startswith("route-set:"):
	187	return
	188
	189	# role (ignored)
	190	elif line.startswith("role:"):
	191	return
	192
	193	# key-cert (ignored)
	194	elif line.startswith("key-cert:"):
	195	return
	196
	197	# irt (ignored)
	198	elif line.startswith("irt:"):
199	return
200
201	# Log any unknown blocks
202	else:
203	log.warning("Unknown block:")
204	for line in block:
205	log.warning(line)
206
207	def _parse_autnum_block(self, block):
208	log.debug("Parsing autnum block:")
209
210	autnum = {}
211	for line in block:
212	# Split line
213	key, val = split_line(line)
214
215	if key == "aut-num":
216	m = re.match(r"^(AS\|as)(\d+)", val)
217	if m:
218	autnum["asn"] = m.group(2)
219
220	elif key in ("as-name", "org"):
221	autnum[key] = val
222
223	# Skip empty objects
224	if not autnum:
225	return
226
227	# Insert into database
228	self.db.execute("INSERT INTO autnums(number, name, organization) \
229	VALUES(%s, %s, %s) ON CONFLICT (number) DO UPDATE SET \
230	name = excluded.name, organization = excluded.organization",
231	autnum.get("asn"), autnum.get("as-name"), autnum.get("org"),
232	)
233
234	def _parse_inetnum_block(self, block):
235	inetnum = {}
236	for line in block:
237	# Split line
238	key, val = split_line(line)
239
240	if key == "inetnum":
241	start_address, delim, end_address = val.partition("-")
242
243	# Strip any excess space
244	start_address, end_address = start_address.rstrip(), end_address.strip()
245
246	# Skip invalid blocks
247	if start_address in INVALID_ADDRESSES:
248	return
249
250	# Convert to IP address
251	try:
252	start_address = ipaddress.ip_address(start_address)
253	end_address = ipaddress.ip_address(end_address)
254	except ValueError:
255	log.warning("Could not parse line: %s" % line)
256	return
257
258	# Set prefix to default
259	prefix = 32
260
261	# Count number of addresses in this subnet
262	num_addresses = int(end_address) - int(start_address)
263	if num_addresses:
264	prefix -= math.log(num_addresses, 2)
265
266	inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
267
268	elif key == "inet6num":
269	# Skip invalid blocks
270	if val in INVALID_ADDRESSES:
271	return
272
273	inetnum[key] = val
274
275	elif key == "netname":
276	inetnum[key] = val
277
278	elif key == "country":
279	if val == "UNITED STATES":
280	val = "US"
281
282	inetnum[key] = val.upper()
283
284	elif key == "descr":
285	if key in inetnum:
286	inetnum[key] += "\n%s" % val
287	else:
288	inetnum[key] = val
289
290	# Skip empty objects
291	if not inetnum:
292	return
293
294	network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
295
296	self.db.execute("INSERT INTO inetnums(network, name, country, description) \
297	VALUES(%s, %s, %s, %s) ON CONFLICT (network) DO \
298	UPDATE SET name = excluded.name, country = excluded.country, description = excluded.description",
299	"%s" % network, inetnum.get("netname"), inetnum.get("country"), inetnum.get("descr"),
300	)
301
302	def _parse_org_block(self, block):
303	org = {}
304	for line in block:
305	# Split line
306	key, val = split_line(line)
307
308	if key in ("organisation", "org-name", "country"):
309	org[key] = val
310
311	# Skip empty objects
312	if not org:
313	return
314
315	self.db.execute("INSERT INTO organizations(handle, name, country) \
316	VALUES(%s, %s, %s) ON CONFLICT (handle) DO \
317	UPDATE SET name = excluded.name, country = excluded.country",
318	org.get("organisation"), org.get("org-name"), org.get("country"),
319	)
320
321	def _parse_route_block(self, block):
322	route = {}
323	for line in block:
324	# Split line
325	key, val = split_line(line)
326
327	# Keep any significant data
328	if key in ("route6", "route"):
329	route[key] = val
330
331	elif key == "origin":
332	m = re.match(r"^(AS\|as)(\d+)", val)
333	if m:
334	route["asn"] = m.group(2)
335
336	# Skip empty objects
337	if not route:
338	return
339
340	network = ipaddress.ip_network(route.get("route6") or route.get("route"), strict=False)
341
342	self.db.execute("INSERT INTO routes(network, asn) \
343	VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET asn = excluded.asn",
344	"%s" % network, route.get("asn"),
345	)
346
347
348	def split_line(line):
349	key, colon, val = line.partition(":")
350
351	# Strip any excess space
352	key = key.strip()
353	val = val.strip()
78ff0cf2	354
6ffd06b5	355	return key, val
78ff0cf2 MT	356
	357	def main():
	358	# Run the command line interface
	359	c = CLI()
	360	c.run()
	361
	362	main()