[location/libloc.git] / src / python / location-downloader.in

#!/usr/bin/python3
###############################################################################
#                                                                             #
# libloc - A library to determine the location of someone on the Internet     #
#                                                                             #
# Copyright (C) 2019 IPFire Development Team <info@ipfire.org>                #
#                                                                             #
# This library is free software; you can redistribute it and/or               #
# modify it under the terms of the GNU Lesser General Public                  #
# License as published by the Free Software Foundation; either                #
# version 2.1 of the License, or (at your option) any later version.          #
#                                                                             #
# This library is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
# Lesser General Public License for more details.                             #
#                                                                             #
###############################################################################

import argparse
import datetime
import gettext
import logging
import logging.handlers
import lzma
import os
import random
import shutil
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request

# Load our location module
import location

DATABASE_FILENAME = "test.db.xz"
MIRRORS = (
	"https://location.ipfire.org/databases/",
	"https://people.ipfire.org/~ms/location/",
)

def setup_logging(level=logging.INFO):
	l = logging.getLogger("location-downloader")
	l.setLevel(level)

	# Log to console
	h = logging.StreamHandler()
	h.setLevel(logging.DEBUG)
	l.addHandler(h)

	# Log to syslog
	h = logging.handlers.SysLogHandler(address="/dev/log",
		facility=logging.handlers.SysLogHandler.LOG_DAEMON)
	h.setLevel(logging.INFO)
	l.addHandler(h)

	# Format syslog messages
	formatter = logging.Formatter("location-downloader[%(process)d]: %(message)s")
	h.setFormatter(formatter)

	return l

# Initialise logging
log = setup_logging()

# i18n
def _(singular, plural=None, n=None):
	if plural:
		return gettext.dngettext("libloc", singular, plural, n)

	return gettext.dgettext("libloc", singular)


class Downloader(object):
	def __init__(self, mirrors):
		self.mirrors = list(mirrors)

		# Randomize mirrors
		random.shuffle(self.mirrors)

		# Get proxies from environment
		self.proxies = self._get_proxies()

	def _get_proxies(self):
		proxies = {}

		for protocol in ("https", "http"):
			proxy = os.environ.get("%s_proxy" % protocol, None)

			if proxy:
				proxies[protocol] = proxy

		return proxies

	def _make_request(self, url, baseurl=None, headers={}):
		if baseurl:
			url = urllib.parse.urljoin(baseurl, url)

		req = urllib.request.Request(url, method="GET")

		# Update headers
		headers.update({
			"User-Agent" : "location-downloader/%s" % location.__version__,
		})

		# Set headers
		for header in headers:
			req.add_header(header, headers[header])

		# Set proxies
		for protocol in self.proxies:
			req.set_proxy(self.proxies[protocol], protocol)

		return req

	def _send_request(self, req, **kwargs):
		# Log request headers
		log.debug("HTTP %s Request to %s" % (req.method, req.host))
		log.debug("	URL: %s" % req.full_url)
		log.debug("	Headers:")
		for k, v in req.header_items():
			log.debug("		%s: %s" % (k, v))

		try:
			res = urllib.request.urlopen(req, **kwargs)

		except urllib.error.HTTPError as e:
			# Log response headers
			log.debug("HTTP Response: %s" % e.code)
			log.debug("	Headers:")
			for header in e.headers:
				log.debug("		%s: %s" % (header, e.headers[header]))

			# Raise all other errors
			raise e

		# Log response headers
		log.debug("HTTP Response: %s" % res.code)
		log.debug("	Headers:")
		for k, v in res.getheaders():
			log.debug("		%s: %s" % (k, v))

		return res

	def download(self, url, public_key, timestamp=None, **kwargs):
		headers = {}

		if timestamp:
			headers["If-Modified-Since"] = timestamp.strftime(
				"%a, %d %b %Y %H:%M:%S GMT",
			)

		t = tempfile.NamedTemporaryFile(delete=False)
		with t:
			# Try all mirrors
			for mirror in self.mirrors:
				# Prepare HTTP request
				req = self._make_request(url, baseurl=mirror, headers=headers)

				try:
					with self._send_request(req) as res:
						decompressor = lzma.LZMADecompressor()

						# Read all data
						while True:
							buf = res.read(1024)
							if not buf:
								break

							# Decompress data
							buf = decompressor.decompress(buf)
							if buf:
								t.write(buf)

					# Write all data to disk
					t.flush()

				# Catch decompression errors
				except lzma.LZMAError as e:
					log.warning("Could not decompress downloaded file: %s" % e)
					continue

				except urllib.error.HTTPError as e:
					# The file on the server was too old
					if e.code == 304:
						log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)

					# Log any other HTTP errors
					else:
						log.warning("%s reported: %s" % (mirror, e))

					# Throw away any downloaded content and try again
					t.truncate()

				else:
					# Check if the downloaded database is recent
					if not self._check_database(t, public_key, timestamp):
						log.warning("Downloaded database is outdated. Trying next mirror...")

						# Throw away the data and try again
						t.truncate()
						continue

					# Return temporary file
					return t

		raise FileNotFoundError(url)

	def _check_database(self, f, public_key, timestamp=None):
		"""
			Checks the downloaded database if it can be opened,
			verified and if it is recent enough
		"""
		log.debug("Opening downloaded database at %s" % f.name)

		db = location.Database(f.name)

		# Database is not recent
		if timestamp and db.created_at < timestamp.timestamp():
			return False

		log.info("Downloaded new database from %s" % (time.strftime(
			"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
		)))

		# Verify the database
		with open(public_key, "r") as f:
			if not db.verify(f):
				log.error("Could not verify database")
				return False

		return True


class CLI(object):
	def __init__(self):
		self.downloader = Downloader(mirrors=MIRRORS)

	def parse_cli(self):
		parser = argparse.ArgumentParser(
			description=_("Location Downloader Command Line Interface"),
		)
		subparsers = parser.add_subparsers()

		# Global configuration flags
		parser.add_argument("--debug", action="store_true",
			help=_("Enable debug output"))

		# version
		parser.add_argument("--version", action="version",
			version="%%(prog)s %s" % location.__version__)

		# database
		parser.add_argument("--database", "-d",
			default="@databasedir@/database.db", help=_("Path to database"),
		)

		# public key
		parser.add_argument("--public-key", "-k",
			default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
		)

		# Update
		update = subparsers.add_parser("update", help=_("Update database"))
		update.set_defaults(func=self.handle_update)

		args = parser.parse_args()

		# Enable debug logging
		if args.debug:
			log.setLevel(logging.DEBUG)

		# Print usage if no action was given
		if not "func" in args:
			parser.print_usage()
			sys.exit(2)

		return args

	def run(self):
		# Parse command line arguments
		args = self.parse_cli()

		# Call function
		ret = args.func(args)

		# Return with exit code
		if ret:
			sys.exit(ret)

		# Otherwise just exit
		sys.exit(0)

	def handle_update(self, ns):
		# Fetch the version we need from DNS
		t = location.discover_latest_version()

		# Parse timestamp into datetime format
		try:
			timestamp = datetime.datetime.fromtimestamp(t)
		except:
			raise

		# Open database
		try:
			db = location.Database(ns.database)

			# Check if we are already on the latest version
			if db.created_at >= timestamp.timestamp():
				log.info("Already on the latest version")
				return

		except FileNotFoundError as e:
			db = None

		# Try downloading a new database
		try:
			t = self.downloader.download(DATABASE_FILENAME,
				public_key=ns.public_key, timestamp=timestamp)

		# If no file could be downloaded, log a message
		except FileNotFoundError as e:
			log.error("Could not download a new database")
			return 1

		# If we have not received a new file, there is nothing to do
		if not t:
			return 0

		# Write temporary file to destination
		shutil.copyfile(t.name, ns.database)

		# Remove temporary file
		os.unlink(t.name)


def main():
	# Run the command line interface
	c = CLI()
	c.run()

main()
Commit	Line	Data
244a3b61 MT	1	#!/usr/bin/python3
	2	###############################################################################
	3	# #
	4	# libloc - A library to determine the location of someone on the Internet #
	5	# #
	6	# Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
	7	# #
	8	# This library is free software; you can redistribute it and/or #
	9	# modify it under the terms of the GNU Lesser General Public #
	10	# License as published by the Free Software Foundation; either #
	11	# version 2.1 of the License, or (at your option) any later version. #
	12	# #
	13	# This library is distributed in the hope that it will be useful, #
	14	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	15	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
	16	# Lesser General Public License for more details. #
	17	# #
	18	###############################################################################
	19
	20	import argparse
f4fef543	21	import datetime
244a3b61	22	import gettext
5a9b4c77 MT	23	import logging
5a9b4c77 MT	24	import logging.handlers
244a3b61 MT	25	import lzma
	26	import os
	27	import random
	28	import shutil
	29	import sys
	30	import tempfile
	31	import time
	32	import urllib.error
	33	import urllib.parse
	34	import urllib.request
	35
	36	# Load our location module
	37	import location
	38
244a3b61 MT	39	DATABASE_FILENAME = "test.db.xz"
	40	MIRRORS = (
	41	"https://location.ipfire.org/databases/",
	42	"https://people.ipfire.org/~ms/location/",
	43	)
	44
5a9b4c77 MT	45	def setup_logging(level=logging.INFO):
	46	l = logging.getLogger("location-downloader")
	47	l.setLevel(level)
	48
	49	# Log to console
	50	h = logging.StreamHandler()
	51	h.setLevel(logging.DEBUG)
	52	l.addHandler(h)
	53
	54	# Log to syslog
	55	h = logging.handlers.SysLogHandler(address="/dev/log",
	56	facility=logging.handlers.SysLogHandler.LOG_DAEMON)
	57	h.setLevel(logging.INFO)
	58	l.addHandler(h)
	59
	60	# Format syslog messages
	61	formatter = logging.Formatter("location-downloader[%(process)d]: %(message)s")
	62	h.setFormatter(formatter)
	63
	64	return l
	65
	66	# Initialise logging
	67	log = setup_logging()
	68
244a3b61 MT	69	# i18n
	70	def _(singular, plural=None, n=None):
	71	if plural:
	72	return gettext.dngettext("libloc", singular, plural, n)
	73
	74	return gettext.dgettext("libloc", singular)
	75
244a3b61 MT	76
	77	class Downloader(object):
	78	def __init__(self, mirrors):
	79	self.mirrors = list(mirrors)
	80
	81	# Randomize mirrors
	82	random.shuffle(self.mirrors)
	83
	84	# Get proxies from environment
	85	self.proxies = self._get_proxies()
	86
	87	def _get_proxies(self):
	88	proxies = {}
	89
	90	for protocol in ("https", "http"):
	91	proxy = os.environ.get("%s_proxy" % protocol, None)
	92
	93	if proxy:
	94	proxies[protocol] = proxy
	95
	96	return proxies
	97
	98	def _make_request(self, url, baseurl=None, headers={}):
	99	if baseurl:
	100	url = urllib.parse.urljoin(baseurl, url)
	101
	102	req = urllib.request.Request(url, method="GET")
	103
	104	# Update headers
	105	headers.update({
	106	"User-Agent" : "location-downloader/%s" % location.__version__,
	107	})
	108
	109	# Set headers
	110	for header in headers:
	111	req.add_header(header, headers[header])
	112
	113	# Set proxies
	114	for protocol in self.proxies:
	115	req.set_proxy(self.proxies[protocol], protocol)
	116
	117	return req
	118
	119	def _send_request(self, req, **kwargs):
	120	# Log request headers
5a9b4c77 MT	121	log.debug("HTTP %s Request to %s" % (req.method, req.host))
	122	log.debug(" URL: %s" % req.full_url)
	123	log.debug(" Headers:")
244a3b61	124	for k, v in req.header_items():
5a9b4c77	125	log.debug(" %s: %s" % (k, v))
244a3b61 MT	126
	127	try:
	128	res = urllib.request.urlopen(req, **kwargs)
	129
	130	except urllib.error.HTTPError as e:
	131	# Log response headers
5a9b4c77 MT	132	log.debug("HTTP Response: %s" % e.code)
5a9b4c77 MT	133	log.debug(" Headers:")
244a3b61	134	for header in e.headers:
5a9b4c77	135	log.debug(" %s: %s" % (header, e.headers[header]))
244a3b61	136
244a3b61 MT	137	# Raise all other errors
	138	raise e
	139
	140	# Log response headers
5a9b4c77 MT	141	log.debug("HTTP Response: %s" % res.code)
5a9b4c77 MT	142	log.debug(" Headers:")
244a3b61	143	for k, v in res.getheaders():
5a9b4c77	144	log.debug(" %s: %s" % (k, v))
244a3b61 MT	145
	146	return res
	147
116b1352	148	def download(self, url, public_key, timestamp=None, **kwargs):
244a3b61 MT	149	headers = {}
244a3b61 MT	150
f4fef543 MT	151	if timestamp:
	152	headers["If-Modified-Since"] = timestamp.strftime(
	153	"%a, %d %b %Y %H:%M:%S GMT",
244a3b61 MT	154	)
	155
	156	t = tempfile.NamedTemporaryFile(delete=False)
	157	with t:
	158	# Try all mirrors
	159	for mirror in self.mirrors:
	160	# Prepare HTTP request
	161	req = self._make_request(url, baseurl=mirror, headers=headers)
	162
	163	try:
	164	with self._send_request(req) as res:
	165	decompressor = lzma.LZMADecompressor()
	166
	167	# Read all data
	168	while True:
	169	buf = res.read(1024)
	170	if not buf:
	171	break
	172
	173	# Decompress data
	174	buf = decompressor.decompress(buf)
	175	if buf:
	176	t.write(buf)
	177
f4fef543 MT	178	# Write all data to disk
f4fef543 MT	179	t.flush()
244a3b61 MT	180
	181	# Catch decompression errors
	182	except lzma.LZMAError as e:
5a9b4c77	183	log.warning("Could not decompress downloaded file: %s" % e)
244a3b61 MT	184	continue
244a3b61 MT	185
244a3b61	186	except urllib.error.HTTPError as e:
f4fef543 MT	187	# The file on the server was too old
	188	if e.code == 304:
	189	log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)
244a3b61	190
f4fef543 MT	191	# Log any other HTTP errors
	192	else:
	193	log.warning("%s reported: %s" % (mirror, e))
	194
	195	# Throw away any downloaded content and try again
	196	t.truncate()
244a3b61	197
f4fef543 MT	198	else:
f4fef543 MT	199	# Check if the downloaded database is recent
116b1352	200	if not self._check_database(t, public_key, timestamp):
f4fef543	201	log.warning("Downloaded database is outdated. Trying next mirror...")
244a3b61	202
f4fef543 MT	203	# Throw away the data and try again
	204	t.truncate()
	205	continue
	206
	207	# Return temporary file
	208	return t
244a3b61 MT	209
	210	raise FileNotFoundError(url)
	211
116b1352	212	def _check_database(self, f, public_key, timestamp=None):
f4fef543 MT	213	"""
	214	Checks the downloaded database if it can be opened,
	215	verified and if it is recent enough
	216	"""
	217	log.debug("Opening downloaded database at %s" % f.name)
	218
	219	db = location.Database(f.name)
	220
	221	# Database is not recent
	222	if timestamp and db.created_at < timestamp.timestamp():
	223	return False
	224
	225	log.info("Downloaded new database from %s" % (time.strftime(
	226	"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
	227	)))
	228
116b1352 MT	229	# Verify the database
	230	with open(public_key, "r") as f:
	231	if not db.verify(f):
	232	log.error("Could not verify database")
	233	return False
	234
f4fef543 MT	235	return True
f4fef543 MT	236
244a3b61 MT	237
	238	class CLI(object):
	239	def __init__(self):
	240	self.downloader = Downloader(mirrors=MIRRORS)
	241
	242	def parse_cli(self):
	243	parser = argparse.ArgumentParser(
	244	description=_("Location Downloader Command Line Interface"),
	245	)
	246	subparsers = parser.add_subparsers()
	247
	248	# Global configuration flags
	249	parser.add_argument("--debug", action="store_true",
	250	help=_("Enable debug output"))
	251
	252	# version
	253	parser.add_argument("--version", action="version",
	254	version="%%(prog)s %s" % location.__version__)
	255
	256	# database
	257	parser.add_argument("--database", "-d",
	258	default="@databasedir@/database.db", help=_("Path to database"),
	259	)
	260
116b1352 MT	261	# public key
	262	parser.add_argument("--public-key", "-k",
	263	default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
	264	)
	265
244a3b61 MT	266	# Update
	267	update = subparsers.add_parser("update", help=_("Update database"))
	268	update.set_defaults(func=self.handle_update)
	269
	270	args = parser.parse_args()
	271
	272	# Enable debug logging
	273	if args.debug:
5a9b4c77	274	log.setLevel(logging.DEBUG)
244a3b61 MT	275
	276	# Print usage if no action was given
	277	if not "func" in args:
	278	parser.print_usage()
	279	sys.exit(2)
	280
	281	return args
	282
	283	def run(self):
	284	# Parse command line arguments
	285	args = self.parse_cli()
	286
	287	# Call function
	288	ret = args.func(args)
	289
	290	# Return with exit code
	291	if ret:
	292	sys.exit(ret)
	293
	294	# Otherwise just exit
	295	sys.exit(0)
	296
	297	def handle_update(self, ns):
f4fef543 MT	298	# Fetch the version we need from DNS
	299	t = location.discover_latest_version()
	300
	301	# Parse timestamp into datetime format
	302	try:
	303	timestamp = datetime.datetime.fromtimestamp(t)
	304	except:
	305	raise
244a3b61 MT	306
	307	# Open database
	308	try:
	309	db = location.Database(ns.database)
	310
f4fef543 MT	311	# Check if we are already on the latest version
	312	if db.created_at >= timestamp.timestamp():
	313	log.info("Already on the latest version")
	314	return
	315
244a3b61 MT	316	except FileNotFoundError as e:
	317	db = None
	318
	319	# Try downloading a new database
	320	try:
116b1352 MT	321	t = self.downloader.download(DATABASE_FILENAME,
116b1352 MT	322	public_key=ns.public_key, timestamp=timestamp)
244a3b61 MT	323
	324	# If no file could be downloaded, log a message
	325	except FileNotFoundError as e:
5a9b4c77	326	log.error("Could not download a new database")
244a3b61 MT	327	return 1
	328
	329	# If we have not received a new file, there is nothing to do
	330	if not t:
	331	return 0
	332
244a3b61 MT	333	# Write temporary file to destination
	334	shutil.copyfile(t.name, ns.database)
	335
	336	# Remove temporary file
	337	os.unlink(t.name)
	338
	339
	340	def main():
	341	# Run the command line interface
	342	c = CLI()
	343	c.run()
	344
	345	main()