[people/ms/libloc.git] / src / python / location-downloader.in

#!/usr/bin/python3
###############################################################################
#                                                                             #
# libloc - A library to determine the location of someone on the Internet     #
#                                                                             #
# Copyright (C) 2019 IPFire Development Team <info@ipfire.org>                #
#                                                                             #
# This library is free software; you can redistribute it and/or               #
# modify it under the terms of the GNU Lesser General Public                  #
# License as published by the Free Software Foundation; either                #
# version 2.1 of the License, or (at your option) any later version.          #
#                                                                             #
# This library is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
# Lesser General Public License for more details.                             #
#                                                                             #
###############################################################################

import argparse
import datetime
import logging
import lzma
import os
import random
import shutil
import stat
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request

# Load our location module
import location
from location.i18n import _

DATABASE_FILENAME = "location.db.xz"
MIRRORS = (
	"https://location.ipfire.org/databases/",
)

# Initialise logging
log = logging.getLogger("location.downloader")
log.propagate = 1

class Downloader(object):
	def __init__(self, version, mirrors):
		self.version = version
		self.mirrors = list(mirrors)

		# Randomize mirrors
		random.shuffle(self.mirrors)

		# Get proxies from environment
		self.proxies = self._get_proxies()

	def _get_proxies(self):
		proxies = {}

		for protocol in ("https", "http"):
			proxy = os.environ.get("%s_proxy" % protocol, None)

			if proxy:
				proxies[protocol] = proxy

		return proxies

	def _make_request(self, url, baseurl=None, headers={}):
		if baseurl:
			url = urllib.parse.urljoin(baseurl, url)

		req = urllib.request.Request(url, method="GET")

		# Update headers
		headers.update({
			"User-Agent" : "location-downloader/@VERSION@",
		})

		# Set headers
		for header in headers:
			req.add_header(header, headers[header])

		# Set proxies
		for protocol in self.proxies:
			req.set_proxy(self.proxies[protocol], protocol)

		return req

	def _send_request(self, req, **kwargs):
		# Log request headers
		log.debug("HTTP %s Request to %s" % (req.method, req.host))
		log.debug("	URL: %s" % req.full_url)
		log.debug("	Headers:")
		for k, v in req.header_items():
			log.debug("		%s: %s" % (k, v))

		try:
			res = urllib.request.urlopen(req, **kwargs)

		except urllib.error.HTTPError as e:
			# Log response headers
			log.debug("HTTP Response: %s" % e.code)
			log.debug("	Headers:")
			for header in e.headers:
				log.debug("		%s: %s" % (header, e.headers[header]))

			# Raise all other errors
			raise e

		# Log response headers
		log.debug("HTTP Response: %s" % res.code)
		log.debug("	Headers:")
		for k, v in res.getheaders():
			log.debug("		%s: %s" % (k, v))

		return res

	def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs):
		headers = {}

		if timestamp:
			headers["If-Modified-Since"] = timestamp.strftime(
				"%a, %d %b %Y %H:%M:%S GMT",
			)

		t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
		with t:
			# Try all mirrors
			for mirror in self.mirrors:
				# Prepare HTTP request
				req = self._make_request(url, baseurl=mirror, headers=headers)

				try:
					with self._send_request(req) as res:
						decompressor = lzma.LZMADecompressor()

						# Read all data
						while True:
							buf = res.read(1024)
							if not buf:
								break

							# Decompress data
							buf = decompressor.decompress(buf)
							if buf:
								t.write(buf)

					# Write all data to disk
					t.flush()

				# Catch decompression errors
				except lzma.LZMAError as e:
					log.warning("Could not decompress downloaded file: %s" % e)
					continue

				except urllib.error.HTTPError as e:
					# The file on the server was too old
					if e.code == 304:
						log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)

					# Log any other HTTP errors
					else:
						log.warning("%s reported: %s" % (mirror, e))

					# Throw away any downloaded content and try again
					t.truncate()

				else:
					# Check if the downloaded database is recent
					if not self._check_database(t, public_key, timestamp):
						log.warning("Downloaded database is outdated. Trying next mirror...")

						# Throw away the data and try again
						t.truncate()
						continue

					# Make the file readable for everyone
					os.chmod(t.name, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)

					# Return temporary file
					return t

		raise FileNotFoundError(url)

	def _check_database(self, f, public_key, timestamp=None):
		"""
			Checks the downloaded database if it can be opened,
			verified and if it is recent enough
		"""
		log.debug("Opening downloaded database at %s" % f.name)

		db = location.Database(f.name)

		# Database is not recent
		if timestamp and db.created_at < timestamp.timestamp():
			return False

		log.info("Downloaded new database from %s" % (time.strftime(
			"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
		)))

		# Verify the database
		with open(public_key, "r") as f:
			if not db.verify(f):
				log.error("Could not verify database")
				return False

		return True


class CLI(object):
	def __init__(self):
		# Which version are we downloading?
		self.version = location.DATABASE_VERSION_LATEST

		self.downloader = Downloader(version=self.version, mirrors=MIRRORS)

	def parse_cli(self):
		parser = argparse.ArgumentParser(
			description=_("Location Downloader Command Line Interface"),
		)
		subparsers = parser.add_subparsers()

		# Global configuration flags
		parser.add_argument("--debug", action="store_true",
			help=_("Enable debug output"))
		parser.add_argument("--quiet", action="store_true",
			help=_("Enable quiet mode"))

		# version
		parser.add_argument("--version", action="version",
			version="%(prog)s @VERSION@")

		# database
		parser.add_argument("--database", "-d",
			default="@databasedir@/database.db", help=_("Path to database"),
		)

		# public key
		parser.add_argument("--public-key", "-k",
			default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
		)

		# Update
		update = subparsers.add_parser("update", help=_("Update database"))
		update.set_defaults(func=self.handle_update)

		# Verify
		verify = subparsers.add_parser("verify",
			help=_("Verify the downloaded database"))
		verify.set_defaults(func=self.handle_verify)

		args = parser.parse_args()

		# Configure logging
		if args.debug:
			location.logger.set_level(logging.DEBUG)
		elif args.quiet:
			location.logger.set_level(logging.WARNING)

		# Print usage if no action was given
		if not "func" in args:
			parser.print_usage()
			sys.exit(2)

		return args

	def run(self):
		# Parse command line arguments
		args = self.parse_cli()

		# Call function
		ret = args.func(args)

		# Return with exit code
		if ret:
			sys.exit(ret)

		# Otherwise just exit
		sys.exit(0)

	def handle_update(self, ns):
		# Fetch the timestamp we need from DNS
		t = location.discover_latest_version(self.version)

		# Parse timestamp into datetime format
		timestamp = datetime.datetime.fromtimestamp(t) if t else None

		# Open database
		try:
			db = location.Database(ns.database)

			# Check if we are already on the latest version
			if timestamp and db.created_at >= timestamp.timestamp():
				log.info("Already on the latest version")
				return

		except FileNotFoundError as e:
			db = None

		# Download the database into the correct directory
		tmpdir = os.path.dirname(ns.database)

		# Try downloading a new database
		try:
			t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME),
				public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)

		# If no file could be downloaded, log a message
		except FileNotFoundError as e:
			log.error("Could not download a new database")
			return 1

		# If we have not received a new file, there is nothing to do
		if not t:
			return 3

		# Move temporary file to destination
		shutil.move(t.name, ns.database)

		return 0

	def handle_verify(self, ns):
		try:
			db = location.Database(ns.database)
		except FileNotFoundError as e:
			log.error("%s: %s" % (ns.database, e))
			return 127

		# Verify the database
		with open(ns.public_key, "r") as f:
			if not db.verify(f):
				log.error("Could not verify database")
				return 1

		# Success
		log.debug("Database successfully verified")
		return 0


def main():
	# Run the command line interface
	c = CLI()
	c.run()

main()
Commit	Line	Data
244a3b61 MT	1	#!/usr/bin/python3
	2	###############################################################################
	3	# #
	4	# libloc - A library to determine the location of someone on the Internet #
	5	# #
	6	# Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
	7	# #
	8	# This library is free software; you can redistribute it and/or #
	9	# modify it under the terms of the GNU Lesser General Public #
	10	# License as published by the Free Software Foundation; either #
	11	# version 2.1 of the License, or (at your option) any later version. #
	12	# #
	13	# This library is distributed in the hope that it will be useful, #
	14	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	15	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
	16	# Lesser General Public License for more details. #
	17	# #
	18	###############################################################################
	19
	20	import argparse
f4fef543	21	import datetime
5a9b4c77	22	import logging
244a3b61 MT	23	import lzma
	24	import os
	25	import random
	26	import shutil
679e5ae2	27	import stat
244a3b61 MT	28	import sys
	29	import tempfile
	30	import time
	31	import urllib.error
	32	import urllib.parse
	33	import urllib.request
	34
	35	# Load our location module
	36	import location
7dccb767	37	from location.i18n import _
244a3b61	38
8881d5f4	39	DATABASE_FILENAME = "location.db.xz"
244a3b61 MT	40	MIRRORS = (
244a3b61 MT	41	"https://location.ipfire.org/databases/",
244a3b61 MT	42	)
244a3b61 MT	43
5a9b4c77	44	# Initialise logging
e44b30f4 MT	45	log = logging.getLogger("location.downloader")
e44b30f4 MT	46	log.propagate = 1
5a9b4c77	47
244a3b61	48	class Downloader(object):
3ee36b5e MT	49	def __init__(self, version, mirrors):
3ee36b5e MT	50	self.version = version
244a3b61 MT	51	self.mirrors = list(mirrors)
	52
	53	# Randomize mirrors
	54	random.shuffle(self.mirrors)
	55
	56	# Get proxies from environment
	57	self.proxies = self._get_proxies()
	58
	59	def _get_proxies(self):
	60	proxies = {}
	61
	62	for protocol in ("https", "http"):
	63	proxy = os.environ.get("%s_proxy" % protocol, None)
	64
	65	if proxy:
	66	proxies[protocol] = proxy
	67
	68	return proxies
	69
	70	def _make_request(self, url, baseurl=None, headers={}):
	71	if baseurl:
	72	url = urllib.parse.urljoin(baseurl, url)
	73
	74	req = urllib.request.Request(url, method="GET")
	75
	76	# Update headers
	77	headers.update({
d2714e4a	78	"User-Agent" : "location-downloader/@VERSION@",
244a3b61 MT	79	})
	80
	81	# Set headers
	82	for header in headers:
	83	req.add_header(header, headers[header])
	84
	85	# Set proxies
	86	for protocol in self.proxies:
	87	req.set_proxy(self.proxies[protocol], protocol)
	88
	89	return req
	90
	91	def _send_request(self, req, **kwargs):
	92	# Log request headers
5a9b4c77 MT	93	log.debug("HTTP %s Request to %s" % (req.method, req.host))
	94	log.debug(" URL: %s" % req.full_url)
	95	log.debug(" Headers:")
244a3b61	96	for k, v in req.header_items():
5a9b4c77	97	log.debug(" %s: %s" % (k, v))
244a3b61 MT	98
	99	try:
	100	res = urllib.request.urlopen(req, **kwargs)
	101
	102	except urllib.error.HTTPError as e:
	103	# Log response headers
5a9b4c77 MT	104	log.debug("HTTP Response: %s" % e.code)
5a9b4c77 MT	105	log.debug(" Headers:")
244a3b61	106	for header in e.headers:
5a9b4c77	107	log.debug(" %s: %s" % (header, e.headers[header]))
244a3b61	108
244a3b61 MT	109	# Raise all other errors
	110	raise e
	111
	112	# Log response headers
5a9b4c77 MT	113	log.debug("HTTP Response: %s" % res.code)
5a9b4c77 MT	114	log.debug(" Headers:")
244a3b61	115	for k, v in res.getheaders():
5a9b4c77	116	log.debug(" %s: %s" % (k, v))
244a3b61 MT	117
	118	return res
	119
679e5ae2	120	def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs):
244a3b61 MT	121	headers = {}
244a3b61 MT	122
f4fef543 MT	123	if timestamp:
	124	headers["If-Modified-Since"] = timestamp.strftime(
	125	"%a, %d %b %Y %H:%M:%S GMT",
244a3b61 MT	126	)
244a3b61 MT	127
679e5ae2	128	t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
244a3b61 MT	129	with t:
	130	# Try all mirrors
	131	for mirror in self.mirrors:
	132	# Prepare HTTP request
	133	req = self._make_request(url, baseurl=mirror, headers=headers)
	134
	135	try:
	136	with self._send_request(req) as res:
	137	decompressor = lzma.LZMADecompressor()
	138
	139	# Read all data
	140	while True:
	141	buf = res.read(1024)
	142	if not buf:
	143	break
	144
	145	# Decompress data
	146	buf = decompressor.decompress(buf)
	147	if buf:
	148	t.write(buf)
	149
f4fef543 MT	150	# Write all data to disk
f4fef543 MT	151	t.flush()
244a3b61 MT	152
	153	# Catch decompression errors
	154	except lzma.LZMAError as e:
5a9b4c77	155	log.warning("Could not decompress downloaded file: %s" % e)
244a3b61 MT	156	continue
244a3b61 MT	157
244a3b61	158	except urllib.error.HTTPError as e:
f4fef543 MT	159	# The file on the server was too old
	160	if e.code == 304:
	161	log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)
244a3b61	162
f4fef543 MT	163	# Log any other HTTP errors
	164	else:
	165	log.warning("%s reported: %s" % (mirror, e))
	166
	167	# Throw away any downloaded content and try again
	168	t.truncate()
244a3b61	169
f4fef543 MT	170	else:
f4fef543 MT	171	# Check if the downloaded database is recent
116b1352	172	if not self._check_database(t, public_key, timestamp):
f4fef543	173	log.warning("Downloaded database is outdated. Trying next mirror...")
244a3b61	174
f4fef543 MT	175	# Throw away the data and try again
	176	t.truncate()
	177	continue
	178
679e5ae2 MT	179	# Make the file readable for everyone
	180	os.chmod(t.name, stat.S_IRUSR\|stat.S_IRGRP\|stat.S_IROTH)
	181
f4fef543 MT	182	# Return temporary file
f4fef543 MT	183	return t
244a3b61 MT	184
	185	raise FileNotFoundError(url)
	186
116b1352	187	def _check_database(self, f, public_key, timestamp=None):
f4fef543 MT	188	"""
	189	Checks the downloaded database if it can be opened,
	190	verified and if it is recent enough
	191	"""
	192	log.debug("Opening downloaded database at %s" % f.name)
	193
	194	db = location.Database(f.name)
	195
	196	# Database is not recent
	197	if timestamp and db.created_at < timestamp.timestamp():
	198	return False
	199
	200	log.info("Downloaded new database from %s" % (time.strftime(
	201	"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
	202	)))
	203
116b1352 MT	204	# Verify the database
	205	with open(public_key, "r") as f:
	206	if not db.verify(f):
	207	log.error("Could not verify database")
	208	return False
	209
f4fef543 MT	210	return True
f4fef543 MT	211
244a3b61 MT	212
	213	class CLI(object):
	214	def __init__(self):
3ee36b5e MT	215	# Which version are we downloading?
	216	self.version = location.DATABASE_VERSION_LATEST
	217
	218	self.downloader = Downloader(version=self.version, mirrors=MIRRORS)
244a3b61 MT	219
	220	def parse_cli(self):
	221	parser = argparse.ArgumentParser(
	222	description=_("Location Downloader Command Line Interface"),
	223	)
	224	subparsers = parser.add_subparsers()
	225
	226	# Global configuration flags
	227	parser.add_argument("--debug", action="store_true",
	228	help=_("Enable debug output"))
bc1f5f53 MT	229	parser.add_argument("--quiet", action="store_true",
bc1f5f53 MT	230	help=_("Enable quiet mode"))
244a3b61 MT	231
	232	# version
	233	parser.add_argument("--version", action="version",
d2714e4a	234	version="%(prog)s @VERSION@")
244a3b61 MT	235
	236	# database
	237	parser.add_argument("--database", "-d",
	238	default="@databasedir@/database.db", help=_("Path to database"),
	239	)
	240
116b1352 MT	241	# public key
	242	parser.add_argument("--public-key", "-k",
	243	default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
	244	)
	245
244a3b61 MT	246	# Update
	247	update = subparsers.add_parser("update", help=_("Update database"))
	248	update.set_defaults(func=self.handle_update)
	249
8e753500 MT	250	# Verify
	251	verify = subparsers.add_parser("verify",
	252	help=_("Verify the downloaded database"))
	253	verify.set_defaults(func=self.handle_verify)
	254
244a3b61 MT	255	args = parser.parse_args()
244a3b61 MT	256
bc1f5f53	257	# Configure logging
244a3b61	258	if args.debug:
f9de5e61	259	location.logger.set_level(logging.DEBUG)
bc1f5f53 MT	260	elif args.quiet:
bc1f5f53 MT	261	location.logger.set_level(logging.WARNING)
244a3b61 MT	262
	263	# Print usage if no action was given
	264	if not "func" in args:
	265	parser.print_usage()
	266	sys.exit(2)
	267
	268	return args
	269
	270	def run(self):
	271	# Parse command line arguments
	272	args = self.parse_cli()
	273
	274	# Call function
	275	ret = args.func(args)
	276
	277	# Return with exit code
	278	if ret:
	279	sys.exit(ret)
	280
	281	# Otherwise just exit
	282	sys.exit(0)
	283
	284	def handle_update(self, ns):
3ee36b5e MT	285	# Fetch the timestamp we need from DNS
3ee36b5e MT	286	t = location.discover_latest_version(self.version)
f4fef543 MT	287
f4fef543 MT	288	# Parse timestamp into datetime format
3ee36b5e	289	timestamp = datetime.datetime.fromtimestamp(t) if t else None
244a3b61 MT	290
	291	# Open database
	292	try:
	293	db = location.Database(ns.database)
	294
f4fef543	295	# Check if we are already on the latest version
3ee36b5e	296	if timestamp and db.created_at >= timestamp.timestamp():
f4fef543 MT	297	log.info("Already on the latest version")
	298	return
	299
244a3b61 MT	300	except FileNotFoundError as e:
	301	db = None
	302
679e5ae2 MT	303	# Download the database into the correct directory
	304	tmpdir = os.path.dirname(ns.database)
	305
244a3b61 MT	306	# Try downloading a new database
244a3b61 MT	307	try:
3ee36b5e	308	t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME),
679e5ae2	309	public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
244a3b61 MT	310
	311	# If no file could be downloaded, log a message
	312	except FileNotFoundError as e:
5a9b4c77	313	log.error("Could not download a new database")
244a3b61 MT	314	return 1
	315
	316	# If we have not received a new file, there is nothing to do
	317	if not t:
f47a500f	318	return 3
244a3b61	319
679e5ae2 MT	320	# Move temporary file to destination
679e5ae2 MT	321	shutil.move(t.name, ns.database)
244a3b61	322
8e753500 MT	323	return 0
	324
	325	def handle_verify(self, ns):
	326	try:
	327	db = location.Database(ns.database)
	328	except FileNotFoundError as e:
	329	log.error("%s: %s" % (ns.database, e))
	330	return 127
	331
	332	# Verify the database
	333	with open(ns.public_key, "r") as f:
	334	if not db.verify(f):
	335	log.error("Could not verify database")
	336	return 1
	337
	338	# Success
	339	log.debug("Database successfully verified")
	340	return 0
f47a500f	341
244a3b61 MT	342
	343	def main():
	344	# Run the command line interface
	345	c = CLI()
	346	c.run()
	347
	348	main()