src/web/base.py

   1 #!/usr/bin/python
   2
   3 import asyncio
   4 import base64
   5 import datetime
   6 import dateutil.parser
   7 import functools
   8 import http.client
   9 import ipaddress
  10 import logging
  11 import magic
  12 import mimetypes
  13 import time
  14 import tornado.locale
  15 import tornado.web
  16
  17 from ..decorators import *
  18 from .. import util
  19
  20 # Setup logging
  21 log = logging.getLogger(__name__)
  22
  23 class ratelimit(object):
  24         """
  25                 A decorator class which limits how often a function can be called
  26         """
  27         def __init__(self, *, minutes, requests):
  28                 self.minutes  = minutes
  29                 self.requests = requests
  30
  31         def __call__(self, method):
  32                 @functools.wraps(method)
  33                 async def wrapper(handler, *args, **kwargs):
  34                         # Pass the request to the rate limiter and get a request object
  35                         req = handler.backend.ratelimiter.handle_request(handler.request,
  36                                 handler, minutes=self.minutes, limit=self.requests)
  37
  38                         # If the rate limit has been reached, we won't allow
  39                         # processing the request and therefore send HTTP error code 429.
  40                         if await req.is_ratelimited():
  41                                 raise tornado.web.HTTPError(429, "Rate limit exceeded")
  42
  43                         # Call the wrapped method
  44                         result = method(handler, *args, **kwargs)
  45
  46                         # Await it if it is a coroutine
  47                         if asyncio.iscoroutine(result):
  48                                 return await result
  49
  50                         # Return the result
  51                         return result
  52
  53                 return wrapper
  54
  55
  56 class BaseHandler(tornado.web.RequestHandler):
  57         def prepare(self):
  58                 # Mark this as private when someone is logged in
  59                 if self.current_user:
  60                         self.set_header("Cache-Control", "private")
  61
  62                 # Always send Vary: Cookie
  63                 self.set_header("Vary", "Cookie")
  64
  65         def set_expires(self, seconds):
  66                 # For HTTP/1.1
  67                 self.add_header("Cache-Control", "max-age=%s, must-revalidate" % seconds)
  68
  69                 # For HTTP/1.0
  70                 expires = datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds)
  71                 self.set_header("Expires", expires)
  72
  73         def write_error(self, status_code, **kwargs):
  74                 # Translate code into message
  75                 try:
  76                         message = http.client.responses[status_code]
  77                 except KeyError:
  78                         message = None
  79
  80                 self.render("error.html", status_code=status_code, message=message, **kwargs)
  81
  82         def browser_accepts(self, t):
  83                 """
  84                         Checks if type is in Accept: header
  85                 """
  86                 accepts = []
  87
  88                 for elem in self.request.headers.get("Accept", "").split(","):
  89                         # Remove q=N
  90                         type, delim, q = elem.partition(";")
  91
  92                         accepts.append(type)
  93
  94                 # Check if the filetype is in the list of accepted ones
  95                 return t in accepts
  96
  97         @property
  98         def hostname(self):
  99                 # Return hostname in production
 100                 if self.request.host.endswith("ipfire.org"):
 101                         return self.request.host
 102
 103                 # Remove the development prefix
 104                 subdomain, delimier, domain = self.request.host.partition(".")
 105                 if subdomain:
 106                         return "%s.ipfire.org" % subdomain
 107
 108                 # Return whatever it is
 109                 return self.request.host
 110
 111         def get_template_namespace(self):
 112                 ns = tornado.web.RequestHandler.get_template_namespace(self)
 113
 114                 now = datetime.date.today()
 115
 116                 ns.update({
 117                         "backend"     : self.backend,
 118                         "debug"       : self.application.settings.get("debug", False),
 119                         "format_size" : util.format_size,
 120                         "format_time" : util.format_time,
 121                         "hostname"    : self.hostname,
 122                         "now"         : now,
 123                         "q"           : None,
 124                         "year"        : now.year,
 125                 })
 126
 127                 return ns
 128
 129         def get_remote_ip(self):
 130                 # Fix for clients behind a proxy that sends "X-Forwarded-For".
 131                 remote_ips = self.request.remote_ip.split(", ")
 132
 133                 for remote_ip in remote_ips:
 134                         try:
 135                                 addr = ipaddress.ip_address(remote_ip)
 136                         except ValueError:
 137                                 # Skip invalid IP addresses.
 138                                 continue
 139
 140                         # Check if the given IP address is from a
 141                         # private network.
 142                         if addr.is_private:
 143                                 continue
 144
 145                         return remote_ip
 146
 147                 # Return the last IP if nothing else worked
 148                 return remote_ips.pop()
 149
 150         @lazy_property
 151         def current_address(self):
 152                 address = self.get_remote_ip()
 153
 154                 if address:
 155                         return util.Address(self.backend, address)
 156
 157         @lazy_property
 158         def current_country_code(self):
 159                 if self.current_address:
 160                         return self.current_address.country_code
 161
 162         @property
 163         def user_agent(self):
 164                 """
 165                         Returns the HTTP user agent
 166                 """
 167                 return self.request.headers.get("User-Agent", None)
 168
 169         @property
 170         def referrer(self):
 171                 return self.request.headers.get("Referer", None)
 172
 173         def _request_basic_authentication(self):
 174                 """
 175                         Called to ask the client to perform HTTP Basic authentication
 176                 """
 177                 # Ask for authentication
 178                 self.set_status(401)
 179
 180                 # Say that we support Basic
 181                 self.set_header("WWW-Authenticate", "Basic realm=Restricted")
 182
 183                 self.finish()
 184
 185         def perform_basic_authentication(self):
 186                 """
 187                         This handles HTTP Basic authentication.
 188                 """
 189                 # Fetch credentials
 190                 cred = self.request.headers.get("Authorization", None)
 191                 if not cred:
 192                         return self._request_basic_authentication()
 193
 194                 # No basic auth? We cannot handle that
 195                 if not cred.startswith("Basic "):
 196                         return self._request_basic_authentication()
 197
 198                 # Decode the credentials
 199                 try:
 200                         # Convert into bytes()
 201                         cred = cred[6:].encode()
 202
 203                         # Decode base64
 204                         cred = base64.b64decode(cred).decode()
 205
 206                         username, password = cred.split(":", 1)
 207
 208                 # Fail if any of those steps failed
 209                 except:
 210                         raise e
 211                         raise tornado.web.HTTPError(400, "Authorization data was malformed")
 212
 213                 # Find the user in the database
 214                 return self.backend.accounts.auth(username, password)
 215
 216                 # Log something
 217                 if account:
 218                         log.info("%s authenticated successfully using HTTP Basic authentication" % account.uid)
 219                 else:
 220                         log.warning("Could not authenticate %s" % username)
 221
 222                 return account
 223
 224         def get_argument_int(self, *args, **kwargs):
 225                 arg = self.get_argument(*args, **kwargs)
 226
 227                 if arg is None or arg == "":
 228                         return
 229
 230                 try:
 231                         return int(arg)
 232                 except ValueError:
 233                         raise tornado.web.HTTPError(400, "Could not convert integer: %s" % arg)
 234
 235         def get_argument_float(self, *args, **kwargs):
 236                 arg = self.get_argument(*args, **kwargs)
 237
 238                 if arg is None or arg == "":
 239                         return
 240
 241                 try:
 242                         return float(arg)
 243                 except ValueError:
 244                         raise tornado.web.HTTPError(400, "Could not convert float: %s" % arg)
 245
 246         def get_argument_date(self, arg, *args, **kwargs):
 247                 value = self.get_argument(arg, *args, **kwargs)
 248                 if value is None:
 249                         return
 250
 251                 try:
 252                         return dateutil.parser.parse(value)
 253                 except ValueError:
 254                         raise tornado.web.HTTPError(400)
 255
 256         def get_file(self, name):
 257                 try:
 258                         file = self.request.files[name][0]
 259
 260                         return file["filename"], file["body"], file["content_type"]
 261                 except KeyError:
 262                         return None
 263
 264         # Initialize libmagic
 265         magic = magic.Magic(mime=True, uncompress=True)
 266
 267         # File
 268
 269         def _deliver_file(self, data, filename=None, prefix=None):
 270                 # Guess content type
 271                 mimetype = self.magic.from_buffer(data)
 272
 273                 # Send the mimetype
 274                 self.set_header("Content-Type", mimetype or "application/octet-stream")
 275
 276                 # Fetch the file extension
 277                 if not filename and prefix:
 278                         ext = mimetypes.guess_extension(mimetype)
 279
 280                         # Compose a new filename
 281                         filename = "%s%s" % (prefix, ext)
 282
 283                 # Set filename
 284                 if filename:
 285                         self.set_header("Content-Disposition", "inline; filename=\"%s\"" % filename)
 286
 287                 # Set size
 288                 if data:
 289                         self.set_header("Content-Length", len(data))
 290
 291                 # Deliver payload
 292                 self.finish(data)
 293
 294         # Login stuff
 295
 296         def get_current_user(self):
 297                 session_id = self.get_cookie("session_id")
 298                 if not session_id:
 299                         return
 300
 301                 # Get account from the session object
 302                 account = self.backend.accounts.get_by_session(session_id, self.request.host)
 303
 304                 # If the account was not found or the session was not valid
 305                 # any more, we will remove the cookie.
 306                 if not account:
 307                         self.clear_cookie("session_id")
 308
 309                 return account
 310
 311         @property
 312         def backend(self):
 313                 return self.application.backend
 314
 315         @property
 316         def db(self):
 317                 return self.backend.db
 318
 319         @property
 320         def accounts(self):
 321                 return self.backend.accounts
 322
 323         @property
 324         def downloads(self):
 325                 return self.backend.downloads
 326
 327         @property
 328         def fireinfo(self):
 329                 return self.backend.fireinfo
 330
 331         @property
 332         def iuse(self):
 333                 return self.backend.iuse
 334
 335         @property
 336         def mirrors(self):
 337                 return self.backend.mirrors
 338
 339         @property
 340         def netboot(self):
 341                 return self.backend.netboot
 342
 343         @property
 344         def releases(self):
 345                 return self.backend.releases
 346
 347
 348 class AnalyticsMixin(object):
 349         def on_finish(self):
 350                 """
 351                         Collect some data about this request
 352                 """
 353                 # Log something
 354                 log.debug("Analytics for %s:" % self)
 355                 log.debug("  User-Agent: %s" % self.user_agent)
 356                 log.debug("  Referrer  : %s" % self.referrer)
 357
 358                 # Do nothing if this requst should be ignored
 359                 if self._ignore_analytics():
 360                         return
 361
 362                 with self.db.transaction():
 363                         # Log unique visits
 364                         self.backend.analytics.log_unique_visit(
 365                                 address=self.current_address,
 366                                 referrer=self.referrer,
 367                                 country_code=self.current_country_code,
 368                                 user_agent=self.user_agent,
 369                                 host=self.request.host,
 370                                 uri=self.request.uri,
 371
 372                                 # UTMs
 373                                 source=self.get_argument("utm_source", None),
 374                                 medium=self.get_argument("utm_medium", None),
 375                                 campaign=self.get_argument("utm_campaign", None),
 376                                 content=self.get_argument("utm_content", None),
 377                                 term=self.get_argument("utm_term", None),
 378
 379                                 # Search queries
 380                                 q=self.get_argument("q", None),
 381                         )
 382
 383         def _ignore_analytics(self):
 384                 """
 385                         Checks if this request should be ignored
 386                 """
 387                 ignored_user_agents = (
 388                         "LWP::Simple",
 389                         "check_http",
 390                 )
 391
 392                 # Only log GET requests
 393                 if not self.request.method == "GET":
 394                         return True
 395
 396                 # Ignore everything from matching user agents
 397                 if self.user_agent:
 398                         for ignored_user_agent in ignored_user_agents:
 399                                 if self.user_agent.startswith(ignored_user_agent):
 400                                         return True
 401
 402
 403 class APIHandler(BaseHandler):
 404         def check_xsrf_cookie(self):
 405                 """
 406                         Do nothing here, because we cannot verify the XSRF token
 407                 """
 408                 pass
 409
 410         def prepare(self):
 411                 # Do not cache any API communication
 412                 self.set_header("Cache-Control", "no-cache")
 413
 414
 415 class NotFoundHandler(BaseHandler):
 416         def prepare(self):
 417                 # Raises 404 as soon as it is called
 418                 raise tornado.web.HTTPError(404)
 419
 420
 421 class ErrorHandler(BaseHandler):
 422         """
 423                 Raises any error we want
 424         """
 425         def get(self, code):
 426                 try:
 427                         code = int(code)
 428                 except:
 429                         raise tornado.web.HTTPError(400)
 430
 431                 raise tornado.web.HTTPError(code)