From: Michael Tremer Date: Sat, 22 Jun 2019 09:08:12 +0000 (+0100) Subject: Deploy rate-limiting X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=372ef1199663398e6010f12dbffc434e4410b4e8;p=ipfire.org.git Deploy rate-limiting To avoid any abuse of various functions of the webapp, we now rate-limit users to a certain number of requests and send them a HTTP 429 error if they exceed that limit. Signed-off-by: Michael Tremer --- diff --git a/Makefile.am b/Makefile.am index f5d759c4..ef2256fc 100644 --- a/Makefile.am +++ b/Makefile.am @@ -64,6 +64,7 @@ backend_PYTHON = \ src/backend/misc.py \ src/backend/netboot.py \ src/backend/nopaste.py \ + src/backend/ratelimit.py \ src/backend/releases.py \ src/backend/settings.py \ src/backend/talk.py \ diff --git a/src/backend/base.py b/src/backend/base.py index f11b5ad5..40cc70a0 100644 --- a/src/backend/base.py +++ b/src/backend/base.py @@ -15,6 +15,7 @@ from . import messages from . import mirrors from . import netboot from . import nopaste +from . import ratelimit from . import releases from . import settings from . import talk @@ -114,6 +115,10 @@ class Backend(object): def messages(self): return messages.Messages(self) + @lazy_property + def ratelimiter(self): + return ratelimit.RateLimiter(self) + @lazy_property def tweets(self): return tweets.Tweets(self) diff --git a/src/backend/memcached.py b/src/backend/memcached.py index f6231aec..e8437d7c 100644 --- a/src/backend/memcached.py +++ b/src/backend/memcached.py @@ -21,6 +21,26 @@ class Memcached(Object): return ret + def get_multi(self, keys, *args, **kwargs): + logging.debug("Retrieving keys from cache: %s" % keys) + + ret = self._connection.get_multi(keys, *args, **kwargs) + + if ret is None: + logging.debug("Found nothing for %s" % keys) + else: + logging.debug("Found object of %s bytes for %s" % (len(ret), keys)) + + return ret + + def add(self, key, data, *args, **kwargs): + if data is None: + logging.debug("Putting nothing into cache for %s" % key) + else: + logging.debug("Putting %s bytes into cache for %s" % (len(data), key)) + + return self._connection.add(key, data, *args, **kwargs) + def set(self, key, data, *args, **kwargs): if data is None: logging.debug("Putting nothing into cache for %s" % key) @@ -31,3 +51,8 @@ class Memcached(Object): def delete(self, key, *args, **kwargs): return self._connection.delete(key, *args, **kwargs) + + def incr(self, key): + logging.debug("Incrementing key %s" % key) + + return self._connection.incr(key) diff --git a/src/backend/ratelimit.py b/src/backend/ratelimit.py new file mode 100644 index 00000000..ec99cc51 --- /dev/null +++ b/src/backend/ratelimit.py @@ -0,0 +1,101 @@ +#!/usr/bin/python + +import datetime + +from . import misc + +class RateLimiter(misc.Object): + def handle_request(self, request, handler, minutes, limit): + return RateLimiterRequest(self.backend, request, handler, + minutes=minutes, limit=limit) + + +class RateLimiterRequest(misc.Object): + prefix = "ratelimit" + + def init(self, request, handler, minutes, limit): + self.request = request + self.handler = handler + + # Save the limits + self.minutes = minutes + self.limit = limit + + self.now = datetime.datetime.utcnow() + + # Fetch the current counter value from the cache + self.counter = self.get_counter() + + # Increment the rate-limiting counter + self.increment_counter() + + # Write the header if we are not limited + if not self.is_ratelimited(): + self.write_headers() + + def is_ratelimited(self): + """ + Returns True if the request is prohibited by the rate limiter + """ + # The client is rate-limited when more requests have been + # received than allowed. + return self.counter >= self.limit + + def get_counter(self): + """ + Returns the number of requests that have been done in + recent time. + """ + keys = self.get_keys_to_check() + + res = self.memcache.get_multi(keys) + if res: + return sum((int(e) for e in res.values())) + + return 0 + + def write_headers(self): + # Send the limit to the user + self.handler.set_header("X-Rate-Limit-Limit", self.limit) + + # Send the user how many requests are left for this time window + self.handler.set_header("X-Rate-Limit-Remaining", + self.limit - self.counter) + + expires = self.now + datetime.timedelta(seconds=self.expires_after) + self.handler.set_header("X-Rate-Limit-Reset", expires.strftime("%s")) + + def get_key(self): + key_prefix = self.get_key_prefix() + + return "%s-%s" % (key_prefix, self.now.strftime("%Y-%m-%d-%H:%M")) + + def get_keys_to_check(self): + key_prefix = self.get_key_prefix() + + keys = [] + for minute in range(self.minutes + 1): + when = self.now - datetime.timedelta(minutes=minute) + + key = "%s-%s" % (key_prefix, when.strftime("%Y-%m-%d-%H:%M")) + keys.append(key) + + return keys + + def get_key_prefix(self): + return "-".join((self.prefix, self.request.host, self.request.path, + self.request.method, self.request.remote_ip,)) + + def increment_counter(self): + key = self.get_key() + + # Add the key or increment if it already exists + if not self.memcache.add(key, "1", self.expires_after): + self.memcache.incr(key) + + @property + def expires_after(self): + """ + Returns the number of seconds after which the counter has reset. + """ + return (self.minutes + 1) * 60 diff --git a/src/web/auth.py b/src/web/auth.py index 931a00a8..bd4ceb2b 100644 --- a/src/web/auth.py +++ b/src/web/auth.py @@ -58,6 +58,7 @@ class LoginHandler(AuthenticationMixin, base.BaseHandler): self.render("auth/login.html", next=next) @base.blacklisted + @base.ratelimit(minutes=60, requests=5) def post(self): username = self.get_argument("username") password = self.get_argument("password") @@ -90,6 +91,7 @@ class RegisterHandler(base.BaseHandler): self.render("auth/register.html") @base.blacklisted + @base.ratelimit(minutes=24*60, requests=5) def post(self): uid = self.get_argument("uid") email = self.get_argument("email") diff --git a/src/web/base.py b/src/web/base.py index f3b0aed3..b92ad74f 100644 --- a/src/web/base.py +++ b/src/web/base.py @@ -30,6 +30,27 @@ def blacklisted(method): return wrapper +class ratelimit(object): + def __init__(self, minutes=15, requests=180): + self.minutes = minutes + self.requests = requests + + def __call__(self, method): + @functools.wraps(method) + def wrapper(handler, *args, **kwargs): + # Pass the request to the rate limiter and get a request object + req = handler.backend.ratelimiter.handle_request(handler.request, + handler, minutes=self.minutes, limit=self.requests) + + # If the rate limit has been reached, we won't allow + # processing the request and therefore send HTTP error code 429. + if req.is_ratelimited(): + raise tornado.web.HTTPError(429, "Rate limit exceeded") + + return method(handler, *args, **kwargs) + + return wrapper + class BaseHandler(tornado.web.RequestHandler): def set_expires(self, seconds): diff --git a/src/web/donate.py b/src/web/donate.py index b1986666..743e5bbc 100644 --- a/src/web/donate.py +++ b/src/web/donate.py @@ -44,6 +44,7 @@ class DonateHandler(base.BaseHandler): amount=amount, currency=currency, frequency=frequency) @tornado.gen.coroutine + @base.ratelimit(minutes=24*60, requests=5) def post(self): amount = self.get_argument("amount") currency = self.get_argument("currency", "EUR") diff --git a/src/web/download.py b/src/web/download.py index 6062524d..ba62ffdc 100644 --- a/src/web/download.py +++ b/src/web/download.py @@ -40,6 +40,7 @@ class FileHandler(base.BaseHandler): self.set_header("Pragma", "no-cache") @base.blacklisted + @base.ratelimit(minutes=5, requests=10) def get(self, filename): mirror = self.backend.mirrors.get_for_download(filename, country_code=self.current_country_code) diff --git a/src/web/newsletter.py b/src/web/newsletter.py index 05f92d73..28fea101 100644 --- a/src/web/newsletter.py +++ b/src/web/newsletter.py @@ -14,6 +14,7 @@ class SubscribeHandler(base.BaseHandler): pass @tornado.gen.coroutine + @base.ratelimit(minutes=15, requests=5) def post(self): address = self.get_argument("email") diff --git a/src/web/nopaste.py b/src/web/nopaste.py index 09a0c3c4..d08eedbc 100644 --- a/src/web/nopaste.py +++ b/src/web/nopaste.py @@ -19,6 +19,7 @@ class CreateHandler(auth.CacheMixin, base.BaseHandler): max_size=self._max_size) @base.blacklisted + @base.ratelimit(minutes=15, requests=5) def post(self): mode = self.get_argument("mode") if not mode in self.MODES: diff --git a/src/web/wiki.py b/src/web/wiki.py index cfbef8c5..30077c07 100644 --- a/src/web/wiki.py +++ b/src/web/wiki.py @@ -67,6 +67,7 @@ class ActionEditHandler(auth.CacheMixin, base.BaseHandler): class ActionUploadHandler(auth.CacheMixin, base.BaseHandler): @tornado.web.authenticated + @base.ratelimit(minutes=60, requests=24) def post(self): path = self.get_argument("path") @@ -92,6 +93,7 @@ class ActionUploadHandler(auth.CacheMixin, base.BaseHandler): class ActionWatchHandler(auth.CacheMixin, base.BaseHandler): @tornado.web.authenticated + @base.ratelimit(minutes=60, requests=180) def get(self, path, action): if path is None: path = "/" @@ -119,6 +121,7 @@ class ActionRenderHandler(auth.CacheMixin, base.BaseHandler): pass # disabled @tornado.web.authenticated + @base.ratelimit(minutes=5, requests=180) def post(self, path): if path is None: path = "/" @@ -262,6 +265,7 @@ class PageHandler(auth.CacheMixin, base.BaseHandler): class SearchHandler(auth.CacheMixin, base.BaseHandler): @base.blacklisted + @base.ratelimit(minutes=15, requests=10) def get(self): q = self.get_argument("q")