[location/libloc.git] / src / python / location-downloader.in

#!/usr/bin/python3
###############################################################################
#                                                                             #
# libloc - A library to determine the location of someone on the Internet     #
#                                                                             #
# Copyright (C) 2019 IPFire Development Team <info@ipfire.org>                #
#                                                                             #
# This library is free software; you can redistribute it and/or               #
# modify it under the terms of the GNU Lesser General Public                  #
# License as published by the Free Software Foundation; either                #
# version 2.1 of the License, or (at your option) any later version.          #
#                                                                             #
# This library is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
# Lesser General Public License for more details.                             #
#                                                                             #
###############################################################################

import argparse
import datetime
import logging
import lzma
import os
import random
import shutil
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request

# Load our location module
import location
from location.i18n import _

DATABASE_FILENAME = "test.db.xz"
MIRRORS = (
	"https://location.ipfire.org/databases/",
	"https://people.ipfire.org/~ms/location/",
)

# Initialise logging
log = logging.getLogger("location.downloader")
log.propagate = 1

class Downloader(object):
	def __init__(self, mirrors):
		self.mirrors = list(mirrors)

		# Randomize mirrors
		random.shuffle(self.mirrors)

		# Get proxies from environment
		self.proxies = self._get_proxies()

	def _get_proxies(self):
		proxies = {}

		for protocol in ("https", "http"):
			proxy = os.environ.get("%s_proxy" % protocol, None)

			if proxy:
				proxies[protocol] = proxy

		return proxies

	def _make_request(self, url, baseurl=None, headers={}):
		if baseurl:
			url = urllib.parse.urljoin(baseurl, url)

		req = urllib.request.Request(url, method="GET")

		# Update headers
		headers.update({
			"User-Agent" : "location-downloader/@VERSION@",
		})

		# Set headers
		for header in headers:
			req.add_header(header, headers[header])

		# Set proxies
		for protocol in self.proxies:
			req.set_proxy(self.proxies[protocol], protocol)

		return req

	def _send_request(self, req, **kwargs):
		# Log request headers
		log.debug("HTTP %s Request to %s" % (req.method, req.host))
		log.debug("	URL: %s" % req.full_url)
		log.debug("	Headers:")
		for k, v in req.header_items():
			log.debug("		%s: %s" % (k, v))

		try:
			res = urllib.request.urlopen(req, **kwargs)

		except urllib.error.HTTPError as e:
			# Log response headers
			log.debug("HTTP Response: %s" % e.code)
			log.debug("	Headers:")
			for header in e.headers:
				log.debug("		%s: %s" % (header, e.headers[header]))

			# Raise all other errors
			raise e

		# Log response headers
		log.debug("HTTP Response: %s" % res.code)
		log.debug("	Headers:")
		for k, v in res.getheaders():
			log.debug("		%s: %s" % (k, v))

		return res

	def download(self, url, public_key, timestamp=None, **kwargs):
		headers = {}

		if timestamp:
			headers["If-Modified-Since"] = timestamp.strftime(
				"%a, %d %b %Y %H:%M:%S GMT",
			)

		t = tempfile.NamedTemporaryFile(delete=False)
		with t:
			# Try all mirrors
			for mirror in self.mirrors:
				# Prepare HTTP request
				req = self._make_request(url, baseurl=mirror, headers=headers)

				try:
					with self._send_request(req) as res:
						decompressor = lzma.LZMADecompressor()

						# Read all data
						while True:
							buf = res.read(1024)
							if not buf:
								break

							# Decompress data
							buf = decompressor.decompress(buf)
							if buf:
								t.write(buf)

					# Write all data to disk
					t.flush()

				# Catch decompression errors
				except lzma.LZMAError as e:
					log.warning("Could not decompress downloaded file: %s" % e)
					continue

				except urllib.error.HTTPError as e:
					# The file on the server was too old
					if e.code == 304:
						log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)

					# Log any other HTTP errors
					else:
						log.warning("%s reported: %s" % (mirror, e))

					# Throw away any downloaded content and try again
					t.truncate()

				else:
					# Check if the downloaded database is recent
					if not self._check_database(t, public_key, timestamp):
						log.warning("Downloaded database is outdated. Trying next mirror...")

						# Throw away the data and try again
						t.truncate()
						continue

					# Return temporary file
					return t

		raise FileNotFoundError(url)

	def _check_database(self, f, public_key, timestamp=None):
		"""
			Checks the downloaded database if it can be opened,
			verified and if it is recent enough
		"""
		log.debug("Opening downloaded database at %s" % f.name)

		db = location.Database(f.name)

		# Database is not recent
		if timestamp and db.created_at < timestamp.timestamp():
			return False

		log.info("Downloaded new database from %s" % (time.strftime(
			"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
		)))

		# Verify the database
		with open(public_key, "r") as f:
			if not db.verify(f):
				log.error("Could not verify database")
				return False

		return True


class CLI(object):
	def __init__(self):
		self.downloader = Downloader(mirrors=MIRRORS)

	def parse_cli(self):
		parser = argparse.ArgumentParser(
			description=_("Location Downloader Command Line Interface"),
		)
		subparsers = parser.add_subparsers()

		# Global configuration flags
		parser.add_argument("--debug", action="store_true",
			help=_("Enable debug output"))

		# version
		parser.add_argument("--version", action="version",
			version="%(prog)s @VERSION@")

		# database
		parser.add_argument("--database", "-d",
			default="@databasedir@/database.db", help=_("Path to database"),
		)

		# public key
		parser.add_argument("--public-key", "-k",
			default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
		)

		# Update
		update = subparsers.add_parser("update", help=_("Update database"))
		update.set_defaults(func=self.handle_update)

		# Verify
		verify = subparsers.add_parser("verify",
			help=_("Verify the downloaded database"))
		verify.set_defaults(func=self.handle_verify)

		args = parser.parse_args()

		# Enable debug logging
		if args.debug:
			location.logger.set_level(logging.DEBUG)

		# Print usage if no action was given
		if not "func" in args:
			parser.print_usage()
			sys.exit(2)

		return args

	def run(self):
		# Parse command line arguments
		args = self.parse_cli()

		# Call function
		ret = args.func(args)

		# Return with exit code
		if ret:
			sys.exit(ret)

		# Otherwise just exit
		sys.exit(0)

	def handle_update(self, ns):
		# Fetch the version we need from DNS
		t = location.discover_latest_version()

		# Parse timestamp into datetime format
		try:
			timestamp = datetime.datetime.fromtimestamp(t)
		except:
			raise

		# Open database
		try:
			db = location.Database(ns.database)

			# Check if we are already on the latest version
			if db.created_at >= timestamp.timestamp():
				log.info("Already on the latest version")
				return

		except FileNotFoundError as e:
			db = None

		# Try downloading a new database
		try:
			t = self.downloader.download(DATABASE_FILENAME,
				public_key=ns.public_key, timestamp=timestamp)

		# If no file could be downloaded, log a message
		except FileNotFoundError as e:
			log.error("Could not download a new database")
			return 1

		# If we have not received a new file, there is nothing to do
		if not t:
			return 3

		# Write temporary file to destination
		shutil.copyfile(t.name, ns.database)

		# Remove temporary file
		os.unlink(t.name)

		return 0

	def handle_verify(self, ns):
		try:
			db = location.Database(ns.database)
		except FileNotFoundError as e:
			log.error("%s: %s" % (ns.database, e))
			return 127

		# Verify the database
		with open(ns.public_key, "r") as f:
			if not db.verify(f):
				log.error("Could not verify database")
				return 1

		# Success
		log.debug("Database successfully verified")
		return 0


def main():
	# Run the command line interface
	c = CLI()
	c.run()

main()
Commit	Line	Data
244a3b61 MT	1	#!/usr/bin/python3
	2	###############################################################################
	3	# #
	4	# libloc - A library to determine the location of someone on the Internet #
	5	# #
	6	# Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
	7	# #
	8	# This library is free software; you can redistribute it and/or #
	9	# modify it under the terms of the GNU Lesser General Public #
	10	# License as published by the Free Software Foundation; either #
	11	# version 2.1 of the License, or (at your option) any later version. #
	12	# #
	13	# This library is distributed in the hope that it will be useful, #
	14	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	15	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
	16	# Lesser General Public License for more details. #
	17	# #
	18	###############################################################################
	19
	20	import argparse
f4fef543	21	import datetime
5a9b4c77	22	import logging
244a3b61 MT	23	import lzma
	24	import os
	25	import random
	26	import shutil
	27	import sys
	28	import tempfile
	29	import time
	30	import urllib.error
	31	import urllib.parse
	32	import urllib.request
	33
	34	# Load our location module
	35	import location
7dccb767	36	from location.i18n import _
244a3b61	37
244a3b61 MT	38	DATABASE_FILENAME = "test.db.xz"
	39	MIRRORS = (
	40	"https://location.ipfire.org/databases/",
	41	"https://people.ipfire.org/~ms/location/",
	42	)
	43
5a9b4c77	44	# Initialise logging
e44b30f4 MT	45	log = logging.getLogger("location.downloader")
e44b30f4 MT	46	log.propagate = 1
5a9b4c77	47
244a3b61 MT	48	class Downloader(object):
	49	def __init__(self, mirrors):
	50	self.mirrors = list(mirrors)
	51
	52	# Randomize mirrors
	53	random.shuffle(self.mirrors)
	54
	55	# Get proxies from environment
	56	self.proxies = self._get_proxies()
	57
	58	def _get_proxies(self):
	59	proxies = {}
	60
	61	for protocol in ("https", "http"):
	62	proxy = os.environ.get("%s_proxy" % protocol, None)
	63
	64	if proxy:
	65	proxies[protocol] = proxy
	66
	67	return proxies
	68
	69	def _make_request(self, url, baseurl=None, headers={}):
	70	if baseurl:
	71	url = urllib.parse.urljoin(baseurl, url)
	72
	73	req = urllib.request.Request(url, method="GET")
	74
	75	# Update headers
	76	headers.update({
d2714e4a	77	"User-Agent" : "location-downloader/@VERSION@",
244a3b61 MT	78	})
	79
	80	# Set headers
	81	for header in headers:
	82	req.add_header(header, headers[header])
	83
	84	# Set proxies
	85	for protocol in self.proxies:
	86	req.set_proxy(self.proxies[protocol], protocol)
	87
	88	return req
	89
	90	def _send_request(self, req, **kwargs):
	91	# Log request headers
5a9b4c77 MT	92	log.debug("HTTP %s Request to %s" % (req.method, req.host))
	93	log.debug(" URL: %s" % req.full_url)
	94	log.debug(" Headers:")
244a3b61	95	for k, v in req.header_items():
5a9b4c77	96	log.debug(" %s: %s" % (k, v))
244a3b61 MT	97
	98	try:
	99	res = urllib.request.urlopen(req, **kwargs)
	100
	101	except urllib.error.HTTPError as e:
	102	# Log response headers
5a9b4c77 MT	103	log.debug("HTTP Response: %s" % e.code)
5a9b4c77 MT	104	log.debug(" Headers:")
244a3b61	105	for header in e.headers:
5a9b4c77	106	log.debug(" %s: %s" % (header, e.headers[header]))
244a3b61	107
244a3b61 MT	108	# Raise all other errors
	109	raise e
	110
	111	# Log response headers
5a9b4c77 MT	112	log.debug("HTTP Response: %s" % res.code)
5a9b4c77 MT	113	log.debug(" Headers:")
244a3b61	114	for k, v in res.getheaders():
5a9b4c77	115	log.debug(" %s: %s" % (k, v))
244a3b61 MT	116
	117	return res
	118
116b1352	119	def download(self, url, public_key, timestamp=None, **kwargs):
244a3b61 MT	120	headers = {}
244a3b61 MT	121
f4fef543 MT	122	if timestamp:
	123	headers["If-Modified-Since"] = timestamp.strftime(
	124	"%a, %d %b %Y %H:%M:%S GMT",
244a3b61 MT	125	)
	126
	127	t = tempfile.NamedTemporaryFile(delete=False)
	128	with t:
	129	# Try all mirrors
	130	for mirror in self.mirrors:
	131	# Prepare HTTP request
	132	req = self._make_request(url, baseurl=mirror, headers=headers)
	133
	134	try:
	135	with self._send_request(req) as res:
	136	decompressor = lzma.LZMADecompressor()
	137
	138	# Read all data
	139	while True:
	140	buf = res.read(1024)
	141	if not buf:
	142	break
	143
	144	# Decompress data
	145	buf = decompressor.decompress(buf)
	146	if buf:
	147	t.write(buf)
	148
f4fef543 MT	149	# Write all data to disk
f4fef543 MT	150	t.flush()
244a3b61 MT	151
	152	# Catch decompression errors
	153	except lzma.LZMAError as e:
5a9b4c77	154	log.warning("Could not decompress downloaded file: %s" % e)
244a3b61 MT	155	continue
244a3b61 MT	156
244a3b61	157	except urllib.error.HTTPError as e:
f4fef543 MT	158	# The file on the server was too old
	159	if e.code == 304:
	160	log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)
244a3b61	161
f4fef543 MT	162	# Log any other HTTP errors
	163	else:
	164	log.warning("%s reported: %s" % (mirror, e))
	165
	166	# Throw away any downloaded content and try again
	167	t.truncate()
244a3b61	168
f4fef543 MT	169	else:
f4fef543 MT	170	# Check if the downloaded database is recent
116b1352	171	if not self._check_database(t, public_key, timestamp):
f4fef543	172	log.warning("Downloaded database is outdated. Trying next mirror...")
244a3b61	173
f4fef543 MT	174	# Throw away the data and try again
	175	t.truncate()
	176	continue
	177
	178	# Return temporary file
	179	return t
244a3b61 MT	180
	181	raise FileNotFoundError(url)
	182
116b1352	183	def _check_database(self, f, public_key, timestamp=None):
f4fef543 MT	184	"""
	185	Checks the downloaded database if it can be opened,
	186	verified and if it is recent enough
	187	"""
	188	log.debug("Opening downloaded database at %s" % f.name)
	189
	190	db = location.Database(f.name)
	191
	192	# Database is not recent
	193	if timestamp and db.created_at < timestamp.timestamp():
	194	return False
	195
	196	log.info("Downloaded new database from %s" % (time.strftime(
	197	"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
	198	)))
	199
116b1352 MT	200	# Verify the database
	201	with open(public_key, "r") as f:
	202	if not db.verify(f):
	203	log.error("Could not verify database")
	204	return False
	205
f4fef543 MT	206	return True
f4fef543 MT	207
244a3b61 MT	208
	209	class CLI(object):
	210	def __init__(self):
	211	self.downloader = Downloader(mirrors=MIRRORS)
	212
	213	def parse_cli(self):
	214	parser = argparse.ArgumentParser(
	215	description=_("Location Downloader Command Line Interface"),
	216	)
	217	subparsers = parser.add_subparsers()
	218
	219	# Global configuration flags
	220	parser.add_argument("--debug", action="store_true",
	221	help=_("Enable debug output"))
	222
	223	# version
	224	parser.add_argument("--version", action="version",
d2714e4a	225	version="%(prog)s @VERSION@")
244a3b61 MT	226
	227	# database
	228	parser.add_argument("--database", "-d",
	229	default="@databasedir@/database.db", help=_("Path to database"),
	230	)
	231
116b1352 MT	232	# public key
	233	parser.add_argument("--public-key", "-k",
	234	default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
	235	)
	236
244a3b61 MT	237	# Update
	238	update = subparsers.add_parser("update", help=_("Update database"))
	239	update.set_defaults(func=self.handle_update)
	240
8e753500 MT	241	# Verify
	242	verify = subparsers.add_parser("verify",
	243	help=_("Verify the downloaded database"))
	244	verify.set_defaults(func=self.handle_verify)
	245
244a3b61 MT	246	args = parser.parse_args()
	247
	248	# Enable debug logging
	249	if args.debug:
f9de5e61	250	location.logger.set_level(logging.DEBUG)
244a3b61 MT	251
	252	# Print usage if no action was given
	253	if not "func" in args:
	254	parser.print_usage()
	255	sys.exit(2)
	256
	257	return args
	258
	259	def run(self):
	260	# Parse command line arguments
	261	args = self.parse_cli()
	262
	263	# Call function
	264	ret = args.func(args)
	265
	266	# Return with exit code
	267	if ret:
	268	sys.exit(ret)
	269
	270	# Otherwise just exit
	271	sys.exit(0)
	272
	273	def handle_update(self, ns):
f4fef543 MT	274	# Fetch the version we need from DNS
	275	t = location.discover_latest_version()
	276
	277	# Parse timestamp into datetime format
	278	try:
	279	timestamp = datetime.datetime.fromtimestamp(t)
	280	except:
	281	raise
244a3b61 MT	282
	283	# Open database
	284	try:
	285	db = location.Database(ns.database)
	286
f4fef543 MT	287	# Check if we are already on the latest version
	288	if db.created_at >= timestamp.timestamp():
	289	log.info("Already on the latest version")
	290	return
	291
244a3b61 MT	292	except FileNotFoundError as e:
	293	db = None
	294
	295	# Try downloading a new database
	296	try:
116b1352 MT	297	t = self.downloader.download(DATABASE_FILENAME,
116b1352 MT	298	public_key=ns.public_key, timestamp=timestamp)
244a3b61 MT	299
	300	# If no file could be downloaded, log a message
	301	except FileNotFoundError as e:
5a9b4c77	302	log.error("Could not download a new database")
244a3b61 MT	303	return 1
	304
	305	# If we have not received a new file, there is nothing to do
	306	if not t:
f47a500f	307	return 3
244a3b61	308
244a3b61 MT	309	# Write temporary file to destination
	310	shutil.copyfile(t.name, ns.database)
	311
	312	# Remove temporary file
	313	os.unlink(t.name)
	314
8e753500 MT	315	return 0
	316
	317	def handle_verify(self, ns):
	318	try:
	319	db = location.Database(ns.database)
	320	except FileNotFoundError as e:
	321	log.error("%s: %s" % (ns.database, e))
	322	return 127
	323
	324	# Verify the database
	325	with open(ns.public_key, "r") as f:
	326	if not db.verify(f):
	327	log.error("Could not verify database")
	328	return 1
	329
	330	# Success
	331	log.debug("Database successfully verified")
	332	return 0
f47a500f	333
244a3b61 MT	334
	335	def main():
	336	# Run the command line interface
	337	c = CLI()
	338	c.run()
	339
	340	main()