[location/libloc.git] / src / python / location-downloader.in

#!/usr/bin/python3
###############################################################################
#                                                                             #
# libloc - A library to determine the location of someone on the Internet     #
#                                                                             #
# Copyright (C) 2019 IPFire Development Team <info@ipfire.org>                #
#                                                                             #
# This library is free software; you can redistribute it and/or               #
# modify it under the terms of the GNU Lesser General Public                  #
# License as published by the Free Software Foundation; either                #
# version 2.1 of the License, or (at your option) any later version.          #
#                                                                             #
# This library is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
# Lesser General Public License for more details.                             #
#                                                                             #
###############################################################################

import argparse
import datetime
import gettext
import logging
import lzma
import os
import random
import shutil
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request

# Load our location module
import location

DATABASE_FILENAME = "test.db.xz"
MIRRORS = (
	"https://location.ipfire.org/databases/",
	"https://people.ipfire.org/~ms/location/",
)

# Initialise logging
log = logging.getLogger("location.downloader")
log.propagate = 1

# i18n
def _(singular, plural=None, n=None):
	if plural:
		return gettext.dngettext("libloc", singular, plural, n)

	return gettext.dgettext("libloc", singular)


class Downloader(object):
	def __init__(self, mirrors):
		self.mirrors = list(mirrors)

		# Randomize mirrors
		random.shuffle(self.mirrors)

		# Get proxies from environment
		self.proxies = self._get_proxies()

	def _get_proxies(self):
		proxies = {}

		for protocol in ("https", "http"):
			proxy = os.environ.get("%s_proxy" % protocol, None)

			if proxy:
				proxies[protocol] = proxy

		return proxies

	def _make_request(self, url, baseurl=None, headers={}):
		if baseurl:
			url = urllib.parse.urljoin(baseurl, url)

		req = urllib.request.Request(url, method="GET")

		# Update headers
		headers.update({
			"User-Agent" : "location-downloader/@VERSION@",
		})

		# Set headers
		for header in headers:
			req.add_header(header, headers[header])

		# Set proxies
		for protocol in self.proxies:
			req.set_proxy(self.proxies[protocol], protocol)

		return req

	def _send_request(self, req, **kwargs):
		# Log request headers
		log.debug("HTTP %s Request to %s" % (req.method, req.host))
		log.debug("	URL: %s" % req.full_url)
		log.debug("	Headers:")
		for k, v in req.header_items():
			log.debug("		%s: %s" % (k, v))

		try:
			res = urllib.request.urlopen(req, **kwargs)

		except urllib.error.HTTPError as e:
			# Log response headers
			log.debug("HTTP Response: %s" % e.code)
			log.debug("	Headers:")
			for header in e.headers:
				log.debug("		%s: %s" % (header, e.headers[header]))

			# Raise all other errors
			raise e

		# Log response headers
		log.debug("HTTP Response: %s" % res.code)
		log.debug("	Headers:")
		for k, v in res.getheaders():
			log.debug("		%s: %s" % (k, v))

		return res

	def download(self, url, public_key, timestamp=None, **kwargs):
		headers = {}

		if timestamp:
			headers["If-Modified-Since"] = timestamp.strftime(
				"%a, %d %b %Y %H:%M:%S GMT",
			)

		t = tempfile.NamedTemporaryFile(delete=False)
		with t:
			# Try all mirrors
			for mirror in self.mirrors:
				# Prepare HTTP request
				req = self._make_request(url, baseurl=mirror, headers=headers)

				try:
					with self._send_request(req) as res:
						decompressor = lzma.LZMADecompressor()

						# Read all data
						while True:
							buf = res.read(1024)
							if not buf:
								break

							# Decompress data
							buf = decompressor.decompress(buf)
							if buf:
								t.write(buf)

					# Write all data to disk
					t.flush()

				# Catch decompression errors
				except lzma.LZMAError as e:
					log.warning("Could not decompress downloaded file: %s" % e)
					continue

				except urllib.error.HTTPError as e:
					# The file on the server was too old
					if e.code == 304:
						log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)

					# Log any other HTTP errors
					else:
						log.warning("%s reported: %s" % (mirror, e))

					# Throw away any downloaded content and try again
					t.truncate()

				else:
					# Check if the downloaded database is recent
					if not self._check_database(t, public_key, timestamp):
						log.warning("Downloaded database is outdated. Trying next mirror...")

						# Throw away the data and try again
						t.truncate()
						continue

					# Return temporary file
					return t

		raise FileNotFoundError(url)

	def _check_database(self, f, public_key, timestamp=None):
		"""
			Checks the downloaded database if it can be opened,
			verified and if it is recent enough
		"""
		log.debug("Opening downloaded database at %s" % f.name)

		db = location.Database(f.name)

		# Database is not recent
		if timestamp and db.created_at < timestamp.timestamp():
			return False

		log.info("Downloaded new database from %s" % (time.strftime(
			"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
		)))

		# Verify the database
		with open(public_key, "r") as f:
			if not db.verify(f):
				log.error("Could not verify database")
				return False

		return True


class CLI(object):
	def __init__(self):
		self.downloader = Downloader(mirrors=MIRRORS)

	def parse_cli(self):
		parser = argparse.ArgumentParser(
			description=_("Location Downloader Command Line Interface"),
		)
		subparsers = parser.add_subparsers()

		# Global configuration flags
		parser.add_argument("--debug", action="store_true",
			help=_("Enable debug output"))

		# version
		parser.add_argument("--version", action="version",
			version="%(prog)s @VERSION@")

		# database
		parser.add_argument("--database", "-d",
			default="@databasedir@/database.db", help=_("Path to database"),
		)

		# public key
		parser.add_argument("--public-key", "-k",
			default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
		)

		# Update
		update = subparsers.add_parser("update", help=_("Update database"))
		update.set_defaults(func=self.handle_update)

		# Verify
		verify = subparsers.add_parser("verify",
			help=_("Verify the downloaded database"))
		verify.set_defaults(func=self.handle_verify)

		args = parser.parse_args()

		# Enable debug logging
		if args.debug:
			log.setLevel(logging.DEBUG)

		# Print usage if no action was given
		if not "func" in args:
			parser.print_usage()
			sys.exit(2)

		return args

	def run(self):
		# Parse command line arguments
		args = self.parse_cli()

		# Call function
		ret = args.func(args)

		# Return with exit code
		if ret:
			sys.exit(ret)

		# Otherwise just exit
		sys.exit(0)

	def handle_update(self, ns):
		# Fetch the version we need from DNS
		t = location.discover_latest_version()

		# Parse timestamp into datetime format
		try:
			timestamp = datetime.datetime.fromtimestamp(t)
		except:
			raise

		# Open database
		try:
			db = location.Database(ns.database)

			# Check if we are already on the latest version
			if db.created_at >= timestamp.timestamp():
				log.info("Already on the latest version")
				return

		except FileNotFoundError as e:
			db = None

		# Try downloading a new database
		try:
			t = self.downloader.download(DATABASE_FILENAME,
				public_key=ns.public_key, timestamp=timestamp)

		# If no file could be downloaded, log a message
		except FileNotFoundError as e:
			log.error("Could not download a new database")
			return 1

		# If we have not received a new file, there is nothing to do
		if not t:
			return 3

		# Write temporary file to destination
		shutil.copyfile(t.name, ns.database)

		# Remove temporary file
		os.unlink(t.name)

		return 0

	def handle_verify(self, ns):
		try:
			db = location.Database(ns.database)
		except FileNotFoundError as e:
			log.error("%s: %s" % (ns.database, e))
			return 127

		# Verify the database
		with open(ns.public_key, "r") as f:
			if not db.verify(f):
				log.error("Could not verify database")
				return 1

		# Success
		log.debug("Database successfully verified")
		return 0


def main():
	# Run the command line interface
	c = CLI()
	c.run()

main()
Commit	Line	Data
244a3b61 MT	1	#!/usr/bin/python3
	2	###############################################################################
	3	# #
	4	# libloc - A library to determine the location of someone on the Internet #
	5	# #
	6	# Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
	7	# #
	8	# This library is free software; you can redistribute it and/or #
	9	# modify it under the terms of the GNU Lesser General Public #
	10	# License as published by the Free Software Foundation; either #
	11	# version 2.1 of the License, or (at your option) any later version. #
	12	# #
	13	# This library is distributed in the hope that it will be useful, #
	14	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	15	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
	16	# Lesser General Public License for more details. #
	17	# #
	18	###############################################################################
	19
	20	import argparse
f4fef543	21	import datetime
244a3b61	22	import gettext
5a9b4c77	23	import logging
244a3b61 MT	24	import lzma
	25	import os
	26	import random
	27	import shutil
	28	import sys
	29	import tempfile
	30	import time
	31	import urllib.error
	32	import urllib.parse
	33	import urllib.request
	34
	35	# Load our location module
	36	import location
	37
244a3b61 MT	38	DATABASE_FILENAME = "test.db.xz"
	39	MIRRORS = (
	40	"https://location.ipfire.org/databases/",
	41	"https://people.ipfire.org/~ms/location/",
	42	)
	43
5a9b4c77	44	# Initialise logging
e44b30f4 MT	45	log = logging.getLogger("location.downloader")
e44b30f4 MT	46	log.propagate = 1
5a9b4c77	47
244a3b61 MT	48	# i18n
	49	def _(singular, plural=None, n=None):
	50	if plural:
	51	return gettext.dngettext("libloc", singular, plural, n)
	52
	53	return gettext.dgettext("libloc", singular)
	54
244a3b61 MT	55
	56	class Downloader(object):
	57	def __init__(self, mirrors):
	58	self.mirrors = list(mirrors)
	59
	60	# Randomize mirrors
	61	random.shuffle(self.mirrors)
	62
	63	# Get proxies from environment
	64	self.proxies = self._get_proxies()
	65
	66	def _get_proxies(self):
	67	proxies = {}
	68
	69	for protocol in ("https", "http"):
	70	proxy = os.environ.get("%s_proxy" % protocol, None)
	71
	72	if proxy:
	73	proxies[protocol] = proxy
	74
	75	return proxies
	76
	77	def _make_request(self, url, baseurl=None, headers={}):
	78	if baseurl:
	79	url = urllib.parse.urljoin(baseurl, url)
	80
	81	req = urllib.request.Request(url, method="GET")
	82
	83	# Update headers
	84	headers.update({
d2714e4a	85	"User-Agent" : "location-downloader/@VERSION@",
244a3b61 MT	86	})
	87
	88	# Set headers
	89	for header in headers:
	90	req.add_header(header, headers[header])
	91
	92	# Set proxies
	93	for protocol in self.proxies:
	94	req.set_proxy(self.proxies[protocol], protocol)
	95
	96	return req
	97
	98	def _send_request(self, req, **kwargs):
	99	# Log request headers
5a9b4c77 MT	100	log.debug("HTTP %s Request to %s" % (req.method, req.host))
	101	log.debug(" URL: %s" % req.full_url)
	102	log.debug(" Headers:")
244a3b61	103	for k, v in req.header_items():
5a9b4c77	104	log.debug(" %s: %s" % (k, v))
244a3b61 MT	105
	106	try:
	107	res = urllib.request.urlopen(req, **kwargs)
	108
	109	except urllib.error.HTTPError as e:
	110	# Log response headers
5a9b4c77 MT	111	log.debug("HTTP Response: %s" % e.code)
5a9b4c77 MT	112	log.debug(" Headers:")
244a3b61	113	for header in e.headers:
5a9b4c77	114	log.debug(" %s: %s" % (header, e.headers[header]))
244a3b61	115
244a3b61 MT	116	# Raise all other errors
	117	raise e
	118
	119	# Log response headers
5a9b4c77 MT	120	log.debug("HTTP Response: %s" % res.code)
5a9b4c77 MT	121	log.debug(" Headers:")
244a3b61	122	for k, v in res.getheaders():
5a9b4c77	123	log.debug(" %s: %s" % (k, v))
244a3b61 MT	124
	125	return res
	126
116b1352	127	def download(self, url, public_key, timestamp=None, **kwargs):
244a3b61 MT	128	headers = {}
244a3b61 MT	129
f4fef543 MT	130	if timestamp:
	131	headers["If-Modified-Since"] = timestamp.strftime(
	132	"%a, %d %b %Y %H:%M:%S GMT",
244a3b61 MT	133	)
	134
	135	t = tempfile.NamedTemporaryFile(delete=False)
	136	with t:
	137	# Try all mirrors
	138	for mirror in self.mirrors:
	139	# Prepare HTTP request
	140	req = self._make_request(url, baseurl=mirror, headers=headers)
	141
	142	try:
	143	with self._send_request(req) as res:
	144	decompressor = lzma.LZMADecompressor()
	145
	146	# Read all data
	147	while True:
	148	buf = res.read(1024)
	149	if not buf:
	150	break
	151
	152	# Decompress data
	153	buf = decompressor.decompress(buf)
	154	if buf:
	155	t.write(buf)
	156
f4fef543 MT	157	# Write all data to disk
f4fef543 MT	158	t.flush()
244a3b61 MT	159
	160	# Catch decompression errors
	161	except lzma.LZMAError as e:
5a9b4c77	162	log.warning("Could not decompress downloaded file: %s" % e)
244a3b61 MT	163	continue
244a3b61 MT	164
244a3b61	165	except urllib.error.HTTPError as e:
f4fef543 MT	166	# The file on the server was too old
	167	if e.code == 304:
	168	log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)
244a3b61	169
f4fef543 MT	170	# Log any other HTTP errors
	171	else:
	172	log.warning("%s reported: %s" % (mirror, e))
	173
	174	# Throw away any downloaded content and try again
	175	t.truncate()
244a3b61	176
f4fef543 MT	177	else:
f4fef543 MT	178	# Check if the downloaded database is recent
116b1352	179	if not self._check_database(t, public_key, timestamp):
f4fef543	180	log.warning("Downloaded database is outdated. Trying next mirror...")
244a3b61	181
f4fef543 MT	182	# Throw away the data and try again
	183	t.truncate()
	184	continue
	185
	186	# Return temporary file
	187	return t
244a3b61 MT	188
	189	raise FileNotFoundError(url)
	190
116b1352	191	def _check_database(self, f, public_key, timestamp=None):
f4fef543 MT	192	"""
	193	Checks the downloaded database if it can be opened,
	194	verified and if it is recent enough
	195	"""
	196	log.debug("Opening downloaded database at %s" % f.name)
	197
	198	db = location.Database(f.name)
	199
	200	# Database is not recent
	201	if timestamp and db.created_at < timestamp.timestamp():
	202	return False
	203
	204	log.info("Downloaded new database from %s" % (time.strftime(
	205	"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
	206	)))
	207
116b1352 MT	208	# Verify the database
	209	with open(public_key, "r") as f:
	210	if not db.verify(f):
	211	log.error("Could not verify database")
	212	return False
	213
f4fef543 MT	214	return True
f4fef543 MT	215
244a3b61 MT	216
	217	class CLI(object):
	218	def __init__(self):
	219	self.downloader = Downloader(mirrors=MIRRORS)
	220
	221	def parse_cli(self):
	222	parser = argparse.ArgumentParser(
	223	description=_("Location Downloader Command Line Interface"),
	224	)
	225	subparsers = parser.add_subparsers()
	226
	227	# Global configuration flags
	228	parser.add_argument("--debug", action="store_true",
	229	help=_("Enable debug output"))
	230
	231	# version
	232	parser.add_argument("--version", action="version",
d2714e4a	233	version="%(prog)s @VERSION@")
244a3b61 MT	234
	235	# database
	236	parser.add_argument("--database", "-d",
	237	default="@databasedir@/database.db", help=_("Path to database"),
	238	)
	239
116b1352 MT	240	# public key
	241	parser.add_argument("--public-key", "-k",
	242	default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
	243	)
	244
244a3b61 MT	245	# Update
	246	update = subparsers.add_parser("update", help=_("Update database"))
	247	update.set_defaults(func=self.handle_update)
	248
8e753500 MT	249	# Verify
	250	verify = subparsers.add_parser("verify",
	251	help=_("Verify the downloaded database"))
	252	verify.set_defaults(func=self.handle_verify)
	253
244a3b61 MT	254	args = parser.parse_args()
	255
	256	# Enable debug logging
	257	if args.debug:
5a9b4c77	258	log.setLevel(logging.DEBUG)
244a3b61 MT	259
	260	# Print usage if no action was given
	261	if not "func" in args:
	262	parser.print_usage()
	263	sys.exit(2)
	264
	265	return args
	266
	267	def run(self):
	268	# Parse command line arguments
	269	args = self.parse_cli()
	270
	271	# Call function
	272	ret = args.func(args)
	273
	274	# Return with exit code
	275	if ret:
	276	sys.exit(ret)
	277
	278	# Otherwise just exit
	279	sys.exit(0)
	280
	281	def handle_update(self, ns):
f4fef543 MT	282	# Fetch the version we need from DNS
	283	t = location.discover_latest_version()
	284
	285	# Parse timestamp into datetime format
	286	try:
	287	timestamp = datetime.datetime.fromtimestamp(t)
	288	except:
	289	raise
244a3b61 MT	290
	291	# Open database
	292	try:
	293	db = location.Database(ns.database)
	294
f4fef543 MT	295	# Check if we are already on the latest version
	296	if db.created_at >= timestamp.timestamp():
	297	log.info("Already on the latest version")
	298	return
	299
244a3b61 MT	300	except FileNotFoundError as e:
	301	db = None
	302
	303	# Try downloading a new database
	304	try:
116b1352 MT	305	t = self.downloader.download(DATABASE_FILENAME,
116b1352 MT	306	public_key=ns.public_key, timestamp=timestamp)
244a3b61 MT	307
	308	# If no file could be downloaded, log a message
	309	except FileNotFoundError as e:
5a9b4c77	310	log.error("Could not download a new database")
244a3b61 MT	311	return 1
	312
	313	# If we have not received a new file, there is nothing to do
	314	if not t:
f47a500f	315	return 3
244a3b61	316
244a3b61 MT	317	# Write temporary file to destination
	318	shutil.copyfile(t.name, ns.database)
	319
	320	# Remove temporary file
	321	os.unlink(t.name)
	322
8e753500 MT	323	return 0
	324
	325	def handle_verify(self, ns):
	326	try:
	327	db = location.Database(ns.database)
	328	except FileNotFoundError as e:
	329	log.error("%s: %s" % (ns.database, e))
	330	return 127
	331
	332	# Verify the database
	333	with open(ns.public_key, "r") as f:
	334	if not db.verify(f):
	335	log.error("Could not verify database")
	336	return 1
	337
	338	# Success
	339	log.debug("Database successfully verified")
	340	return 0
f47a500f	341
244a3b61 MT	342
	343	def main():
	344	# Run the command line interface
	345	c = CLI()
	346	c.run()
	347
	348	main()