[ipfire.org.git] / src / web / base.py

#!/usr/bin/python

import asyncio
import base64
import datetime
import dateutil.parser
import functools
import http.client
import ipaddress
import logging
import magic
import mimetypes
import time
import tornado.locale
import tornado.web

from ..decorators import *
from .. import util

# Setup logging
log = logging.getLogger(__name__)

class ratelimit(object):
	"""
		A decorator class which limits how often a function can be called
	"""
	def __init__(self, *, minutes, requests):
		self.minutes  = minutes
		self.requests = requests

	def __call__(self, method):
		@functools.wraps(method)
		async def wrapper(handler, *args, **kwargs):
			# Pass the request to the rate limiter and get a request object
			req = handler.backend.ratelimiter.handle_request(handler.request,
				handler, minutes=self.minutes, limit=self.requests)

			# If the rate limit has been reached, we won't allow
			# processing the request and therefore send HTTP error code 429.
			if await req.is_ratelimited():
				raise tornado.web.HTTPError(429, "Rate limit exceeded")

			# Call the wrapped method
			result = method(handler, *args, **kwargs)

			# Await it if it is a coroutine
			if asyncio.iscoroutine(result):
				return await result

			# Return the result
			return result

		return wrapper


class BaseHandler(tornado.web.RequestHandler):
	def prepare(self):
		# Mark this as private when someone is logged in
		if self.current_user:
			self.set_header("Cache-Control", "private")

		# Always send Vary: Cookie
		self.set_header("Vary", "Cookie")

	def set_expires(self, seconds):
		# For HTTP/1.1
		self.add_header("Cache-Control", "max-age=%s, must-revalidate" % seconds)

		# For HTTP/1.0
		expires = datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds)
		self.set_header("Expires", expires)

	def write_error(self, status_code, **kwargs):
		# Translate code into message
		try:
			message = http.client.responses[status_code]
		except KeyError:
			message = None

		self.render("error.html", status_code=status_code, message=message, **kwargs)

	def browser_accepts(self, t):
		"""
			Checks if type is in Accept: header
		"""
		accepts = []

		for elem in self.request.headers.get("Accept", "").split(","):
			# Remove q=N
			type, delim, q = elem.partition(";")

			accepts.append(type)

		# Check if the filetype is in the list of accepted ones
		return t in accepts

	@property
	def hostname(self):
		# Return hostname in production
		if self.request.host.endswith("ipfire.org"):
			return self.request.host

		# Remove the development prefix
		subdomain, delimier, domain = self.request.host.partition(".")
		if subdomain:
			return "%s.ipfire.org" % subdomain

		# Return whatever it is
		return self.request.host

	def get_template_namespace(self):
		ns = tornado.web.RequestHandler.get_template_namespace(self)

		now = datetime.date.today()

		ns.update({
			"backend"     : self.backend,
			"debug"       : self.application.settings.get("debug", False),
			"format_size" : util.format_size,
			"format_time" : util.format_time,
			"hostname"    : self.hostname,
			"now"         : now,
			"q"           : None,
			"year"        : now.year,
		})

		return ns

	def get_remote_ip(self):
		# Fix for clients behind a proxy that sends "X-Forwarded-For".
		remote_ips = self.request.remote_ip.split(", ")

		for remote_ip in remote_ips:
			try:
				addr = ipaddress.ip_address(remote_ip)
			except ValueError:
				# Skip invalid IP addresses.
				continue

			# Check if the given IP address is from a
			# private network.
			if addr.is_private:
				continue

			return remote_ip

		# Return the last IP if nothing else worked
		return remote_ips.pop()

	@lazy_property
	def current_address(self):
		address = self.get_remote_ip()

		if address:
			return util.Address(self.backend, address)

	@lazy_property
	def current_country_code(self):
		if self.current_address:
			return self.current_address.country_code

	@property
	def user_agent(self):
		"""
			Returns the HTTP user agent
		"""
		return self.request.headers.get("User-Agent", None)

	@property
	def referrer(self):
		return self.request.headers.get("Referer", None)

	def _request_basic_authentication(self):
		"""
			Called to ask the client to perform HTTP Basic authentication
		"""
		# Ask for authentication
		self.set_status(401)

		# Say that we support Basic
		self.set_header("WWW-Authenticate", "Basic realm=Restricted")

		self.finish()

	def perform_basic_authentication(self):
		"""
			This handles HTTP Basic authentication.
		"""
		# Fetch credentials
		cred = self.request.headers.get("Authorization", None)
		if not cred:
			return self._request_basic_authentication()

		# No basic auth? We cannot handle that
		if not cred.startswith("Basic "):
			return self._request_basic_authentication()

		# Decode the credentials
		try:
			# Convert into bytes()
			cred = cred[6:].encode()

			# Decode base64
			cred = base64.b64decode(cred).decode()

			username, password = cred.split(":", 1)

		# Fail if any of those steps failed
		except:
			raise e
			raise tornado.web.HTTPError(400, "Authorization data was malformed")

		# Find the user in the database
		return self.backend.accounts.auth(username, password)

		# Log something
		if account:
			log.info("%s authenticated successfully using HTTP Basic authentication" % account.uid)
		else:
			log.warning("Could not authenticate %s" % username)

		return account

	def get_argument_int(self, *args, **kwargs):
		arg = self.get_argument(*args, **kwargs)

		if arg is None or arg == "":
			return

		try:
			return int(arg)
		except ValueError:
			raise tornado.web.HTTPError(400, "Could not convert integer: %s" % arg)

	def get_argument_float(self, *args, **kwargs):
		arg = self.get_argument(*args, **kwargs)

		if arg is None or arg == "":
			return

		try:
			return float(arg)
		except ValueError:
			raise tornado.web.HTTPError(400, "Could not convert float: %s" % arg)

	def get_argument_date(self, arg, *args, **kwargs):
		value = self.get_argument(arg, *args, **kwargs)
		if value is None:
			return

		try:
			return dateutil.parser.parse(value)
		except ValueError:
			raise tornado.web.HTTPError(400)

	def get_file(self, name):
		try:
			file = self.request.files[name][0]

			return file["filename"], file["body"], file["content_type"]
		except KeyError:
			return None

	# Initialize libmagic
	magic = magic.Magic(mime=True, uncompress=True)

	# File

	def _deliver_file(self, data, filename=None, prefix=None):
		# Guess content type
		mimetype = self.magic.from_buffer(data)

		# Send the mimetype
		self.set_header("Content-Type", mimetype or "application/octet-stream")

		# Fetch the file extension
		if not filename and prefix:
			ext = mimetypes.guess_extension(mimetype)

			# Compose a new filename
			filename = "%s%s" % (prefix, ext)

		# Set filename
		if filename:
			self.set_header("Content-Disposition", "inline; filename=\"%s\"" % filename)

		# Set size
		if data:
			self.set_header("Content-Length", len(data))

		# Deliver payload
		self.finish(data)

	# Login stuff

	def get_current_user(self):
		session_id = self.get_cookie("session_id")
		if not session_id:
			return

		# Get account from the session object
		account = self.backend.accounts.get_by_session(session_id, self.request.host)

		# If the account was not found or the session was not valid
		# any more, we will remove the cookie.
		if not account:
			self.clear_cookie("session_id")

		return account

	@property
	def backend(self):
		return self.application.backend

	@property
	def db(self):
		return self.backend.db

	@property
	def accounts(self):
		return self.backend.accounts

	@property
	def downloads(self):
		return self.backend.downloads

	@property
	def fireinfo(self):
		return self.backend.fireinfo

	@property
	def iuse(self):
		return self.backend.iuse

	@property
	def mirrors(self):
		return self.backend.mirrors

	@property
	def netboot(self):
		return self.backend.netboot

	@property
	def releases(self):
		return self.backend.releases


class AnalyticsMixin(object):
	def on_finish(self):
		"""
			Collect some data about this request
		"""
		# Log something
		log.debug("Analytics for %s:" % self)
		log.debug("  User-Agent: %s" % self.user_agent)
		log.debug("  Referrer  : %s" % self.referrer)

		# Do nothing if this requst should be ignored
		if self._ignore_analytics():
			return

		with self.db.transaction():
			# Log unique visits
			self.backend.analytics.log_unique_visit(
				address=self.current_address,
				referrer=self.referrer,
				country_code=self.current_country_code,
				user_agent=self.user_agent,
				host=self.request.host,
				uri=self.request.uri,

				# UTMs
				source=self.get_argument("utm_source", None),
				medium=self.get_argument("utm_medium", None),
				campaign=self.get_argument("utm_campaign", None),
				content=self.get_argument("utm_content", None),
				term=self.get_argument("utm_term", None),

				# Search queries
				q=self.get_argument("q", None),
			)

	def _ignore_analytics(self):
		"""
			Checks if this request should be ignored
		"""
		ignored_user_agents = (
			"LWP::Simple",
			"check_http",
		)

		# Only log GET requests
		if not self.request.method == "GET":
			return True

		# Ignore everything from matching user agents
		if self.user_agent:
			for ignored_user_agent in ignored_user_agents:
				if self.user_agent.startswith(ignored_user_agent):
					return True


class APIHandler(BaseHandler):
	def check_xsrf_cookie(self):
		"""
			Do nothing here, because we cannot verify the XSRF token
		"""
		pass

	def prepare(self):
		# Do not cache any API communication
		self.set_header("Cache-Control", "no-cache")


class NotFoundHandler(BaseHandler):
	def prepare(self):
		# Raises 404 as soon as it is called
		raise tornado.web.HTTPError(404)


class ErrorHandler(BaseHandler):
	"""
		Raises any error we want
	"""
	def get(self, code):
		try:
			code = int(code)
		except:
			raise tornado.web.HTTPError(400)

		raise tornado.web.HTTPError(code)
Commit	Line	Data
	1	#!/usr/bin/python
	2
	3	import asyncio
	4	import base64
	5	import datetime
	6	import dateutil.parser
	7	import functools
	8	import http.client
	9	import ipaddress
	10	import logging
	11	import magic
	12	import mimetypes
	13	import time
	14	import tornado.locale
	15	import tornado.web
	16
	17	from ..decorators import *
	18	from .. import util
	19
	20	# Setup logging
	21	log = logging.getLogger(__name__)
	22
	23	class ratelimit(object):
	24	"""
	25	A decorator class which limits how often a function can be called
	26	"""
	27	def __init__(self, *, minutes, requests):
	28	self.minutes = minutes
	29	self.requests = requests
	30
	31	def __call__(self, method):
	32	@functools.wraps(method)
	33	async def wrapper(handler, args, *kwargs):
	34	# Pass the request to the rate limiter and get a request object
	35	req = handler.backend.ratelimiter.handle_request(handler.request,
	36	handler, minutes=self.minutes, limit=self.requests)
	37
	38	# If the rate limit has been reached, we won't allow
	39	# processing the request and therefore send HTTP error code 429.
	40	if await req.is_ratelimited():
	41	raise tornado.web.HTTPError(429, "Rate limit exceeded")
	42
	43	# Call the wrapped method
	44	result = method(handler, args, *kwargs)
	45
	46	# Await it if it is a coroutine
	47	if asyncio.iscoroutine(result):
	48	return await result
	49
	50	# Return the result
	51	return result
	52
	53	return wrapper
	54
	55
	56	class BaseHandler(tornado.web.RequestHandler):
	57	def prepare(self):
	58	# Mark this as private when someone is logged in
	59	if self.current_user:
	60	self.set_header("Cache-Control", "private")
	61
	62	# Always send Vary: Cookie
	63	self.set_header("Vary", "Cookie")
	64
	65	def set_expires(self, seconds):
	66	# For HTTP/1.1
	67	self.add_header("Cache-Control", "max-age=%s, must-revalidate" % seconds)
	68
	69	# For HTTP/1.0
	70	expires = datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds)
	71	self.set_header("Expires", expires)
	72
	73	def write_error(self, status_code, **kwargs):
	74	# Translate code into message
	75	try:
	76	message = http.client.responses[status_code]
	77	except KeyError:
	78	message = None
	79
	80	self.render("error.html", status_code=status_code, message=message, **kwargs)
	81
	82	def browser_accepts(self, t):
	83	"""
	84	Checks if type is in Accept: header
	85	"""
	86	accepts = []
	87
	88	for elem in self.request.headers.get("Accept", "").split(","):
	89	# Remove q=N
	90	type, delim, q = elem.partition(";")
	91
	92	accepts.append(type)
	93
	94	# Check if the filetype is in the list of accepted ones
	95	return t in accepts
	96
	97	@property
	98	def hostname(self):
	99	# Return hostname in production
	100	if self.request.host.endswith("ipfire.org"):
	101	return self.request.host
	102
	103	# Remove the development prefix
	104	subdomain, delimier, domain = self.request.host.partition(".")
	105	if subdomain:
	106	return "%s.ipfire.org" % subdomain
	107
	108	# Return whatever it is
	109	return self.request.host
	110
	111	def get_template_namespace(self):
	112	ns = tornado.web.RequestHandler.get_template_namespace(self)
	113
	114	now = datetime.date.today()
	115
	116	ns.update({
	117	"backend" : self.backend,
	118	"debug" : self.application.settings.get("debug", False),
	119	"format_size" : util.format_size,
	120	"format_time" : util.format_time,
	121	"hostname" : self.hostname,
	122	"now" : now,
	123	"q" : None,
	124	"year" : now.year,
	125	})
	126
	127	return ns
	128
	129	def get_remote_ip(self):
	130	# Fix for clients behind a proxy that sends "X-Forwarded-For".
	131	remote_ips = self.request.remote_ip.split(", ")
	132
	133	for remote_ip in remote_ips:
	134	try:
	135	addr = ipaddress.ip_address(remote_ip)
	136	except ValueError:
	137	# Skip invalid IP addresses.
	138	continue
	139
	140	# Check if the given IP address is from a
	141	# private network.
	142	if addr.is_private:
	143	continue
	144
	145	return remote_ip
	146
	147	# Return the last IP if nothing else worked
	148	return remote_ips.pop()
	149
	150	@lazy_property
	151	def current_address(self):
	152	address = self.get_remote_ip()
	153
	154	if address:
	155	return util.Address(self.backend, address)
	156
	157	@lazy_property
	158	def current_country_code(self):
	159	if self.current_address:
	160	return self.current_address.country_code
	161
	162	@property
	163	def user_agent(self):
	164	"""
	165	Returns the HTTP user agent
	166	"""
	167	return self.request.headers.get("User-Agent", None)
	168
	169	@property
	170	def referrer(self):
	171	return self.request.headers.get("Referer", None)
	172
	173	def _request_basic_authentication(self):
	174	"""
	175	Called to ask the client to perform HTTP Basic authentication
	176	"""
	177	# Ask for authentication
	178	self.set_status(401)
	179
	180	# Say that we support Basic
	181	self.set_header("WWW-Authenticate", "Basic realm=Restricted")
	182
	183	self.finish()
	184
	185	def perform_basic_authentication(self):
	186	"""
	187	This handles HTTP Basic authentication.
	188	"""
	189	# Fetch credentials
	190	cred = self.request.headers.get("Authorization", None)
	191	if not cred:
	192	return self._request_basic_authentication()
	193
	194	# No basic auth? We cannot handle that
	195	if not cred.startswith("Basic "):
	196	return self._request_basic_authentication()
	197
	198	# Decode the credentials
	199	try:
	200	# Convert into bytes()
	201	cred = cred[6:].encode()
	202
	203	# Decode base64
	204	cred = base64.b64decode(cred).decode()
	205
	206	username, password = cred.split(":", 1)
	207
	208	# Fail if any of those steps failed
	209	except:
	210	raise e
	211	raise tornado.web.HTTPError(400, "Authorization data was malformed")
	212
	213	# Find the user in the database
	214	return self.backend.accounts.auth(username, password)
	215
	216	# Log something
	217	if account:
	218	log.info("%s authenticated successfully using HTTP Basic authentication" % account.uid)
	219	else:
	220	log.warning("Could not authenticate %s" % username)
	221
	222	return account
	223
	224	def get_argument_int(self, args, *kwargs):
	225	arg = self.get_argument(args, *kwargs)
	226
	227	if arg is None or arg == "":
	228	return
	229
	230	try:
	231	return int(arg)
	232	except ValueError:
	233	raise tornado.web.HTTPError(400, "Could not convert integer: %s" % arg)
	234
	235	def get_argument_float(self, args, *kwargs):
	236	arg = self.get_argument(args, *kwargs)
	237
	238	if arg is None or arg == "":
	239	return
	240
	241	try:
	242	return float(arg)
	243	except ValueError:
	244	raise tornado.web.HTTPError(400, "Could not convert float: %s" % arg)
	245
	246	def get_argument_date(self, arg, args, *kwargs):
	247	value = self.get_argument(arg, args, *kwargs)
	248	if value is None:
	249	return
	250
	251	try:
	252	return dateutil.parser.parse(value)
	253	except ValueError:
	254	raise tornado.web.HTTPError(400)
	255
	256	def get_file(self, name):
	257	try:
	258	file = self.request.files[name][0]
	259
	260	return file["filename"], file["body"], file["content_type"]
	261	except KeyError:
	262	return None
	263
	264	# Initialize libmagic
	265	magic = magic.Magic(mime=True, uncompress=True)
	266
	267	# File
	268
	269	def _deliver_file(self, data, filename=None, prefix=None):
	270	# Guess content type
	271	mimetype = self.magic.from_buffer(data)
	272
	273	# Send the mimetype
	274	self.set_header("Content-Type", mimetype or "application/octet-stream")
	275
	276	# Fetch the file extension
	277	if not filename and prefix:
	278	ext = mimetypes.guess_extension(mimetype)
	279
	280	# Compose a new filename
	281	filename = "%s%s" % (prefix, ext)
	282
	283	# Set filename
	284	if filename:
	285	self.set_header("Content-Disposition", "inline; filename=\"%s\"" % filename)
	286
	287	# Set size
	288	if data:
	289	self.set_header("Content-Length", len(data))
	290
	291	# Deliver payload
	292	self.finish(data)
	293
	294	# Login stuff
	295
	296	def get_current_user(self):
	297	session_id = self.get_cookie("session_id")
	298	if not session_id:
	299	return
	300
	301	# Get account from the session object
	302	account = self.backend.accounts.get_by_session(session_id, self.request.host)
	303
	304	# If the account was not found or the session was not valid
	305	# any more, we will remove the cookie.
	306	if not account:
	307	self.clear_cookie("session_id")
	308
	309	return account
	310
	311	@property
	312	def backend(self):
	313	return self.application.backend
	314
	315	@property
	316	def db(self):
	317	return self.backend.db
	318
	319	@property
	320	def accounts(self):
	321	return self.backend.accounts
	322
	323	@property
	324	def downloads(self):
	325	return self.backend.downloads
	326
	327	@property
	328	def fireinfo(self):
	329	return self.backend.fireinfo
	330
	331	@property
	332	def iuse(self):
	333	return self.backend.iuse
	334
	335	@property
	336	def mirrors(self):
	337	return self.backend.mirrors
	338
	339	@property
	340	def netboot(self):
	341	return self.backend.netboot
	342
	343	@property
	344	def releases(self):
	345	return self.backend.releases
	346
	347
	348	class AnalyticsMixin(object):
	349	def on_finish(self):
	350	"""
	351	Collect some data about this request
	352	"""
	353	# Log something
	354	log.debug("Analytics for %s:" % self)
	355	log.debug(" User-Agent: %s" % self.user_agent)
	356	log.debug(" Referrer : %s" % self.referrer)
	357
	358	# Do nothing if this requst should be ignored
	359	if self._ignore_analytics():
	360	return
	361
	362	with self.db.transaction():
	363	# Log unique visits
	364	self.backend.analytics.log_unique_visit(
	365	address=self.current_address,
	366	referrer=self.referrer,
	367	country_code=self.current_country_code,
	368	user_agent=self.user_agent,
	369	host=self.request.host,
	370	uri=self.request.uri,
	371
	372	# UTMs
	373	source=self.get_argument("utm_source", None),
	374	medium=self.get_argument("utm_medium", None),
	375	campaign=self.get_argument("utm_campaign", None),
	376	content=self.get_argument("utm_content", None),
	377	term=self.get_argument("utm_term", None),
	378
	379	# Search queries
	380	q=self.get_argument("q", None),
	381	)
	382
	383	def _ignore_analytics(self):
	384	"""
	385	Checks if this request should be ignored
	386	"""
	387	ignored_user_agents = (
	388	"LWP::Simple",
	389	"check_http",
	390	)
	391
	392	# Only log GET requests
	393	if not self.request.method == "GET":
	394	return True
	395
	396	# Ignore everything from matching user agents
	397	if self.user_agent:
	398	for ignored_user_agent in ignored_user_agents:
	399	if self.user_agent.startswith(ignored_user_agent):
	400	return True
	401
	402
	403	class APIHandler(BaseHandler):
	404	def check_xsrf_cookie(self):
	405	"""
	406	Do nothing here, because we cannot verify the XSRF token
	407	"""
	408	pass
	409
	410	def prepare(self):
	411	# Do not cache any API communication
	412	self.set_header("Cache-Control", "no-cache")
	413
	414
	415	class NotFoundHandler(BaseHandler):
	416	def prepare(self):
	417	# Raises 404 as soon as it is called
	418	raise tornado.web.HTTPError(404)
	419
	420
	421	class ErrorHandler(BaseHandler):
	422	"""
	423	Raises any error we want
	424	"""
	425	def get(self, code):
	426	try:
	427	code = int(code)
	428	except:
	429	raise tornado.web.HTTPError(400)
	430
	431	raise tornado.web.HTTPError(code)