[location/libloc.git] / src / python / location-downloader.in

#!/usr/bin/python3
###############################################################################
#                                                                             #
# libloc - A library to determine the location of someone on the Internet     #
#                                                                             #
# Copyright (C) 2019 IPFire Development Team <info@ipfire.org>                #
#                                                                             #
# This library is free software; you can redistribute it and/or               #
# modify it under the terms of the GNU Lesser General Public                  #
# License as published by the Free Software Foundation; either                #
# version 2.1 of the License, or (at your option) any later version.          #
#                                                                             #
# This library is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
# Lesser General Public License for more details.                             #
#                                                                             #
###############################################################################

import argparse
import datetime
import logging
import lzma
import os
import random
import shutil
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request

# Load our location module
import location
from location.i18n import _

DATABASE_FILENAME = "location.db.xz"
MIRRORS = (
	"https://location.ipfire.org/databases/",
)

# Initialise logging
log = logging.getLogger("location.downloader")
log.propagate = 1

class Downloader(object):
	def __init__(self, mirrors):
		self.mirrors = list(mirrors)

		# Randomize mirrors
		random.shuffle(self.mirrors)

		# Get proxies from environment
		self.proxies = self._get_proxies()

	def _get_proxies(self):
		proxies = {}

		for protocol in ("https", "http"):
			proxy = os.environ.get("%s_proxy" % protocol, None)

			if proxy:
				proxies[protocol] = proxy

		return proxies

	def _make_request(self, url, baseurl=None, headers={}):
		if baseurl:
			url = urllib.parse.urljoin(baseurl, url)

		req = urllib.request.Request(url, method="GET")

		# Update headers
		headers.update({
			"User-Agent" : "location-downloader/@VERSION@",
		})

		# Set headers
		for header in headers:
			req.add_header(header, headers[header])

		# Set proxies
		for protocol in self.proxies:
			req.set_proxy(self.proxies[protocol], protocol)

		return req

	def _send_request(self, req, **kwargs):
		# Log request headers
		log.debug("HTTP %s Request to %s" % (req.method, req.host))
		log.debug("	URL: %s" % req.full_url)
		log.debug("	Headers:")
		for k, v in req.header_items():
			log.debug("		%s: %s" % (k, v))

		try:
			res = urllib.request.urlopen(req, **kwargs)

		except urllib.error.HTTPError as e:
			# Log response headers
			log.debug("HTTP Response: %s" % e.code)
			log.debug("	Headers:")
			for header in e.headers:
				log.debug("		%s: %s" % (header, e.headers[header]))

			# Raise all other errors
			raise e

		# Log response headers
		log.debug("HTTP Response: %s" % res.code)
		log.debug("	Headers:")
		for k, v in res.getheaders():
			log.debug("		%s: %s" % (k, v))

		return res

	def download(self, url, public_key, timestamp=None, **kwargs):
		headers = {}

		if timestamp:
			headers["If-Modified-Since"] = timestamp.strftime(
				"%a, %d %b %Y %H:%M:%S GMT",
			)

		t = tempfile.NamedTemporaryFile(delete=False)
		with t:
			# Try all mirrors
			for mirror in self.mirrors:
				# Prepare HTTP request
				req = self._make_request(url, baseurl=mirror, headers=headers)

				try:
					with self._send_request(req) as res:
						decompressor = lzma.LZMADecompressor()

						# Read all data
						while True:
							buf = res.read(1024)
							if not buf:
								break

							# Decompress data
							buf = decompressor.decompress(buf)
							if buf:
								t.write(buf)

					# Write all data to disk
					t.flush()

				# Catch decompression errors
				except lzma.LZMAError as e:
					log.warning("Could not decompress downloaded file: %s" % e)
					continue

				except urllib.error.HTTPError as e:
					# The file on the server was too old
					if e.code == 304:
						log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)

					# Log any other HTTP errors
					else:
						log.warning("%s reported: %s" % (mirror, e))

					# Throw away any downloaded content and try again
					t.truncate()

				else:
					# Check if the downloaded database is recent
					if not self._check_database(t, public_key, timestamp):
						log.warning("Downloaded database is outdated. Trying next mirror...")

						# Throw away the data and try again
						t.truncate()
						continue

					# Return temporary file
					return t

		raise FileNotFoundError(url)

	def _check_database(self, f, public_key, timestamp=None):
		"""
			Checks the downloaded database if it can be opened,
			verified and if it is recent enough
		"""
		log.debug("Opening downloaded database at %s" % f.name)

		db = location.Database(f.name)

		# Database is not recent
		if timestamp and db.created_at < timestamp.timestamp():
			return False

		log.info("Downloaded new database from %s" % (time.strftime(
			"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
		)))

		# Verify the database
		with open(public_key, "r") as f:
			if not db.verify(f):
				log.error("Could not verify database")
				return False

		return True


class CLI(object):
	def __init__(self):
		self.downloader = Downloader(mirrors=MIRRORS)

	def parse_cli(self):
		parser = argparse.ArgumentParser(
			description=_("Location Downloader Command Line Interface"),
		)
		subparsers = parser.add_subparsers()

		# Global configuration flags
		parser.add_argument("--debug", action="store_true",
			help=_("Enable debug output"))
		parser.add_argument("--quiet", action="store_true",
			help=_("Enable quiet mode"))

		# version
		parser.add_argument("--version", action="version",
			version="%(prog)s @VERSION@")

		# database
		parser.add_argument("--database", "-d",
			default="@databasedir@/database.db", help=_("Path to database"),
		)

		# public key
		parser.add_argument("--public-key", "-k",
			default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
		)

		# Update
		update = subparsers.add_parser("update", help=_("Update database"))
		update.set_defaults(func=self.handle_update)

		# Verify
		verify = subparsers.add_parser("verify",
			help=_("Verify the downloaded database"))
		verify.set_defaults(func=self.handle_verify)

		args = parser.parse_args()

		# Configure logging
		if args.debug:
			location.logger.set_level(logging.DEBUG)
		elif args.quiet:
			location.logger.set_level(logging.WARNING)

		# Print usage if no action was given
		if not "func" in args:
			parser.print_usage()
			sys.exit(2)

		return args

	def run(self):
		# Parse command line arguments
		args = self.parse_cli()

		# Call function
		ret = args.func(args)

		# Return with exit code
		if ret:
			sys.exit(ret)

		# Otherwise just exit
		sys.exit(0)

	def handle_update(self, ns):
		# Fetch the version we need from DNS
		t = location.discover_latest_version()

		# Parse timestamp into datetime format
		try:
			timestamp = datetime.datetime.fromtimestamp(t)
		except:
			raise

		# Open database
		try:
			db = location.Database(ns.database)

			# Check if we are already on the latest version
			if db.created_at >= timestamp.timestamp():
				log.info("Already on the latest version")
				return

		except FileNotFoundError as e:
			db = None

		# Try downloading a new database
		try:
			t = self.downloader.download(DATABASE_FILENAME,
				public_key=ns.public_key, timestamp=timestamp)

		# If no file could be downloaded, log a message
		except FileNotFoundError as e:
			log.error("Could not download a new database")
			return 1

		# If we have not received a new file, there is nothing to do
		if not t:
			return 3

		# Write temporary file to destination
		shutil.copyfile(t.name, ns.database)

		# Remove temporary file
		os.unlink(t.name)

		return 0

	def handle_verify(self, ns):
		try:
			db = location.Database(ns.database)
		except FileNotFoundError as e:
			log.error("%s: %s" % (ns.database, e))
			return 127

		# Verify the database
		with open(ns.public_key, "r") as f:
			if not db.verify(f):
				log.error("Could not verify database")
				return 1

		# Success
		log.debug("Database successfully verified")
		return 0


def main():
	# Run the command line interface
	c = CLI()
	c.run()

main()
Commit	Line	Data
244a3b61 MT	1	#!/usr/bin/python3
	2	###############################################################################
	3	# #
	4	# libloc - A library to determine the location of someone on the Internet #
	5	# #
	6	# Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
	7	# #
	8	# This library is free software; you can redistribute it and/or #
	9	# modify it under the terms of the GNU Lesser General Public #
	10	# License as published by the Free Software Foundation; either #
	11	# version 2.1 of the License, or (at your option) any later version. #
	12	# #
	13	# This library is distributed in the hope that it will be useful, #
	14	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	15	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
	16	# Lesser General Public License for more details. #
	17	# #
	18	###############################################################################
	19
	20	import argparse
f4fef543	21	import datetime
5a9b4c77	22	import logging
244a3b61 MT	23	import lzma
	24	import os
	25	import random
	26	import shutil
	27	import sys
	28	import tempfile
	29	import time
	30	import urllib.error
	31	import urllib.parse
	32	import urllib.request
	33
	34	# Load our location module
	35	import location
7dccb767	36	from location.i18n import _
244a3b61	37
8881d5f4	38	DATABASE_FILENAME = "location.db.xz"
244a3b61 MT	39	MIRRORS = (
244a3b61 MT	40	"https://location.ipfire.org/databases/",
244a3b61 MT	41	)
244a3b61 MT	42
5a9b4c77	43	# Initialise logging
e44b30f4 MT	44	log = logging.getLogger("location.downloader")
e44b30f4 MT	45	log.propagate = 1
5a9b4c77	46
244a3b61 MT	47	class Downloader(object):
	48	def __init__(self, mirrors):
	49	self.mirrors = list(mirrors)
	50
	51	# Randomize mirrors
	52	random.shuffle(self.mirrors)
	53
	54	# Get proxies from environment
	55	self.proxies = self._get_proxies()
	56
	57	def _get_proxies(self):
	58	proxies = {}
	59
	60	for protocol in ("https", "http"):
	61	proxy = os.environ.get("%s_proxy" % protocol, None)
	62
	63	if proxy:
	64	proxies[protocol] = proxy
	65
	66	return proxies
	67
	68	def _make_request(self, url, baseurl=None, headers={}):
	69	if baseurl:
	70	url = urllib.parse.urljoin(baseurl, url)
	71
	72	req = urllib.request.Request(url, method="GET")
	73
	74	# Update headers
	75	headers.update({
d2714e4a	76	"User-Agent" : "location-downloader/@VERSION@",
244a3b61 MT	77	})
	78
	79	# Set headers
	80	for header in headers:
	81	req.add_header(header, headers[header])
	82
	83	# Set proxies
	84	for protocol in self.proxies:
	85	req.set_proxy(self.proxies[protocol], protocol)
	86
	87	return req
	88
	89	def _send_request(self, req, **kwargs):
	90	# Log request headers
5a9b4c77 MT	91	log.debug("HTTP %s Request to %s" % (req.method, req.host))
	92	log.debug(" URL: %s" % req.full_url)
	93	log.debug(" Headers:")
244a3b61	94	for k, v in req.header_items():
5a9b4c77	95	log.debug(" %s: %s" % (k, v))
244a3b61 MT	96
	97	try:
	98	res = urllib.request.urlopen(req, **kwargs)
	99
	100	except urllib.error.HTTPError as e:
	101	# Log response headers
5a9b4c77 MT	102	log.debug("HTTP Response: %s" % e.code)
5a9b4c77 MT	103	log.debug(" Headers:")
244a3b61	104	for header in e.headers:
5a9b4c77	105	log.debug(" %s: %s" % (header, e.headers[header]))
244a3b61	106
244a3b61 MT	107	# Raise all other errors
	108	raise e
	109
	110	# Log response headers
5a9b4c77 MT	111	log.debug("HTTP Response: %s" % res.code)
5a9b4c77 MT	112	log.debug(" Headers:")
244a3b61	113	for k, v in res.getheaders():
5a9b4c77	114	log.debug(" %s: %s" % (k, v))
244a3b61 MT	115
	116	return res
	117
116b1352	118	def download(self, url, public_key, timestamp=None, **kwargs):
244a3b61 MT	119	headers = {}
244a3b61 MT	120
f4fef543 MT	121	if timestamp:
	122	headers["If-Modified-Since"] = timestamp.strftime(
	123	"%a, %d %b %Y %H:%M:%S GMT",
244a3b61 MT	124	)
	125
	126	t = tempfile.NamedTemporaryFile(delete=False)
	127	with t:
	128	# Try all mirrors
	129	for mirror in self.mirrors:
	130	# Prepare HTTP request
	131	req = self._make_request(url, baseurl=mirror, headers=headers)
	132
	133	try:
	134	with self._send_request(req) as res:
	135	decompressor = lzma.LZMADecompressor()
	136
	137	# Read all data
	138	while True:
	139	buf = res.read(1024)
	140	if not buf:
	141	break
	142
	143	# Decompress data
	144	buf = decompressor.decompress(buf)
	145	if buf:
	146	t.write(buf)
	147
f4fef543 MT	148	# Write all data to disk
f4fef543 MT	149	t.flush()
244a3b61 MT	150
	151	# Catch decompression errors
	152	except lzma.LZMAError as e:
5a9b4c77	153	log.warning("Could not decompress downloaded file: %s" % e)
244a3b61 MT	154	continue
244a3b61 MT	155
244a3b61	156	except urllib.error.HTTPError as e:
f4fef543 MT	157	# The file on the server was too old
	158	if e.code == 304:
	159	log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)
244a3b61	160
f4fef543 MT	161	# Log any other HTTP errors
	162	else:
	163	log.warning("%s reported: %s" % (mirror, e))
	164
	165	# Throw away any downloaded content and try again
	166	t.truncate()
244a3b61	167
f4fef543 MT	168	else:
f4fef543 MT	169	# Check if the downloaded database is recent
116b1352	170	if not self._check_database(t, public_key, timestamp):
f4fef543	171	log.warning("Downloaded database is outdated. Trying next mirror...")
244a3b61	172
f4fef543 MT	173	# Throw away the data and try again
	174	t.truncate()
	175	continue
	176
	177	# Return temporary file
	178	return t
244a3b61 MT	179
	180	raise FileNotFoundError(url)
	181
116b1352	182	def _check_database(self, f, public_key, timestamp=None):
f4fef543 MT	183	"""
	184	Checks the downloaded database if it can be opened,
	185	verified and if it is recent enough
	186	"""
	187	log.debug("Opening downloaded database at %s" % f.name)
	188
	189	db = location.Database(f.name)
	190
	191	# Database is not recent
	192	if timestamp and db.created_at < timestamp.timestamp():
	193	return False
	194
	195	log.info("Downloaded new database from %s" % (time.strftime(
	196	"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
	197	)))
	198
116b1352 MT	199	# Verify the database
	200	with open(public_key, "r") as f:
	201	if not db.verify(f):
	202	log.error("Could not verify database")
	203	return False
	204
f4fef543 MT	205	return True
f4fef543 MT	206
244a3b61 MT	207
	208	class CLI(object):
	209	def __init__(self):
	210	self.downloader = Downloader(mirrors=MIRRORS)
	211
	212	def parse_cli(self):
	213	parser = argparse.ArgumentParser(
	214	description=_("Location Downloader Command Line Interface"),
	215	)
	216	subparsers = parser.add_subparsers()
	217
	218	# Global configuration flags
	219	parser.add_argument("--debug", action="store_true",
	220	help=_("Enable debug output"))
bc1f5f53 MT	221	parser.add_argument("--quiet", action="store_true",
bc1f5f53 MT	222	help=_("Enable quiet mode"))
244a3b61 MT	223
	224	# version
	225	parser.add_argument("--version", action="version",
d2714e4a	226	version="%(prog)s @VERSION@")
244a3b61 MT	227
	228	# database
	229	parser.add_argument("--database", "-d",
	230	default="@databasedir@/database.db", help=_("Path to database"),
	231	)
	232
116b1352 MT	233	# public key
	234	parser.add_argument("--public-key", "-k",
	235	default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
	236	)
	237
244a3b61 MT	238	# Update
	239	update = subparsers.add_parser("update", help=_("Update database"))
	240	update.set_defaults(func=self.handle_update)
	241
8e753500 MT	242	# Verify
	243	verify = subparsers.add_parser("verify",
	244	help=_("Verify the downloaded database"))
	245	verify.set_defaults(func=self.handle_verify)
	246
244a3b61 MT	247	args = parser.parse_args()
244a3b61 MT	248
bc1f5f53	249	# Configure logging
244a3b61	250	if args.debug:
f9de5e61	251	location.logger.set_level(logging.DEBUG)
bc1f5f53 MT	252	elif args.quiet:
bc1f5f53 MT	253	location.logger.set_level(logging.WARNING)
244a3b61 MT	254
	255	# Print usage if no action was given
	256	if not "func" in args:
	257	parser.print_usage()
	258	sys.exit(2)
	259
	260	return args
	261
	262	def run(self):
	263	# Parse command line arguments
	264	args = self.parse_cli()
	265
	266	# Call function
	267	ret = args.func(args)
	268
	269	# Return with exit code
	270	if ret:
	271	sys.exit(ret)
	272
	273	# Otherwise just exit
	274	sys.exit(0)
	275
	276	def handle_update(self, ns):
f4fef543 MT	277	# Fetch the version we need from DNS
	278	t = location.discover_latest_version()
	279
	280	# Parse timestamp into datetime format
	281	try:
	282	timestamp = datetime.datetime.fromtimestamp(t)
	283	except:
	284	raise
244a3b61 MT	285
	286	# Open database
	287	try:
	288	db = location.Database(ns.database)
	289
f4fef543 MT	290	# Check if we are already on the latest version
	291	if db.created_at >= timestamp.timestamp():
	292	log.info("Already on the latest version")
	293	return
	294
244a3b61 MT	295	except FileNotFoundError as e:
	296	db = None
	297
	298	# Try downloading a new database
	299	try:
116b1352 MT	300	t = self.downloader.download(DATABASE_FILENAME,
116b1352 MT	301	public_key=ns.public_key, timestamp=timestamp)
244a3b61 MT	302
	303	# If no file could be downloaded, log a message
	304	except FileNotFoundError as e:
5a9b4c77	305	log.error("Could not download a new database")
244a3b61 MT	306	return 1
	307
	308	# If we have not received a new file, there is nothing to do
	309	if not t:
f47a500f	310	return 3
244a3b61	311
244a3b61 MT	312	# Write temporary file to destination
	313	shutil.copyfile(t.name, ns.database)
	314
	315	# Remove temporary file
	316	os.unlink(t.name)
	317
8e753500 MT	318	return 0
	319
	320	def handle_verify(self, ns):
	321	try:
	322	db = location.Database(ns.database)
	323	except FileNotFoundError as e:
	324	log.error("%s: %s" % (ns.database, e))
	325	return 127
	326
	327	# Verify the database
	328	with open(ns.public_key, "r") as f:
	329	if not db.verify(f):
	330	log.error("Could not verify database")
	331	return 1
	332
	333	# Success
	334	log.debug("Database successfully verified")
	335	return 0
f47a500f	336
244a3b61 MT	337
	338	def main():
	339	# Run the command line interface
	340	c = CLI()
	341	c.run()
	342
	343	main()