mirrors: Add a universal download load balancer

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/mirrors.py b/src/buildservice/mirrors.py
index c2aabcb57ff99bed505c33813573418ab753ed92..e3b7df06e99d848d0a564c0990ea3bb51a52c957 100644 (file)
--- a/src/buildservice/mirrors.py
+++ b/src/buildservice/mirrors.py
@@ -434,6 +434,68 @@ class Mirror(database.Base, database.BackendMixin, database.SoftDeleteMixin):
                # Run the statement
                return await self.db.select_one(stmt, "uptime")
 
+       async def serves_file(self, path):
+               """
+                       Checks if this mirror serves the file
+               """
+               # XXX Skip this if the mirror is not online
+
+               # Make the URL
+               url = self.make_url(path)
+
+               # Make the cache key
+               cache_key = "file-check:%s" % url
+
+               # Check if we have something in the cache
+               serves_file = await self.backend.cache.get(cache_key)
+
+               # Nothing in cache, let's run the check
+               if serves_file is None:
+                       serves_file = await self._serves_file(url, cache_key=cache_key)
+
+               return serves_file
+
+       async def _serves_file(self, url, cache_key=None):
+               serves_file = None
+
+               # Send a HEAD request for the URL
+               try:
+                       response = await self.backend.httpclient.fetch(
+                               url, method="HEAD", follow_redirects=True,
+
+                               # Don't allow too much time for the mirror to respond
+                               connect_timeout=5, request_timeout=5,
+
+                               # Ensure the server responds to all types or requests
+                               headers = {
+                                       "Accept"        : "*/*",
+                                       "Cache-Control" : "no-cache",
+                               }
+                       )
+
+               # Catch any HTTP errors
+               except tornado.httpclient.HTTPClientError as e:
+                       log.error("Mirror %s returned %s for %s" % (self, e.code, url))
+                       serves_file = False
+
+               # If there was no error, we assume this file can be downloaded
+               else:
+                       log.debug("Mirror %s seems to be serving %s")
+                       serves_file = True
+
+               # Store positive responses in the cache for 24 hours
+               # and negative responses for six hours.
+               if cache_key:
+                       if serves_file:
+                               ttl = 86400 # 24h
+                       else:
+                               ttl = 21600 # 6h
+
+                       # Store in cache
+                       await self.backend.cache.set(cache_key, serves_file, ttl)
+
+               return serves_file
+
 
 class MirrorCheck(database.Base):
        """

diff --git a/src/web/__init__.py b/src/web/__init__.py
index e5a07af7c24d981d7863efc6f9770b6308455e60..91f040db7528cdd8fcbeb2d05202ac8079890584 100644 (file)
--- a/src/web/__init__.py
+++ b/src/web/__init__.py
@@ -169,6 +169,9 @@ class Application(tornado.web.Application):
                        (r"/distros/([A-Za-z0-9\-\.]+)/releases/([\w\-_]+)/edit", distributions.ReleasesEditHandler),
                        (r"/distros/([A-Za-z0-9\-\.]+)/releases/([\w\-_]+)/publish", distributions.ReleasesPublishHandler),
 
+                       # Downloads
+                       (r"/downloads/(.*)", mirrors.DownloadsHandler),
+
                        # Mirrors
                        (r"/mirrors",                                    mirrors.IndexHandler),
                        (r"/mirrors/create",             mirrors.CreateHandler),

diff --git a/src/web/mirrors.py b/src/web/mirrors.py
index f329537a40614268a211578741d50144f586117c..2ec5202875d519b36a44102d4c18b82e0b2bef8d 100644 (file)
--- a/src/web/mirrors.py
+++ b/src/web/mirrors.py
@@ -1,9 +1,13 @@
 #!/usr/bin/python
 
+import logging
 import tornado.web
 
 from . import base
 
+# Setup logging
+log = logging.getLogger("pbs.web.mirrors")
+
 class IndexHandler(base.BaseHandler):
        async def get(self):
                await self.render("mirrors/index.html", mirrors=self.backend.mirrors)
@@ -117,3 +121,40 @@ class DeleteHandler(base.BaseHandler):
 
                # Redirect back to all mirrors
                self.redirect("/mirrors")
+
+
+class DownloadsHandler(base.BaseHandler):
+       """
+               A universal download redirector which does not need to keep any state.
+
+               It will check if the mirror serves the file and redirect the client. We are
+               starting with the closest mirror and walk through all of the until we have
+               found the right file.
+
+               As a last resort, we will try to serve the file locally.
+       """
+       async def get(self, path):
+               # Fetch all mirrors for this client
+               mirrors = await self.backend.mirrors.get_mirrors_for_address(self.current_address)
+
+               # Walk through all mirrors
+               for mirror in mirrors:
+                       # Skip the mirror if it does not serve the file we are looking for
+                       if not await mirror.serves_file(path):
+                               continue
+
+                       # Log action
+                       log.info("Sending %s to download %s from %s", self.current_address, path, mirror)
+
+                       # Make the URL
+                       url = mirror.make_url(path)
+
+                       # Redirect the user
+                       return self.redirect(url)
+
+               # If we go here, we did not find any working mirror.
+               # We will send the user to our local file store and hopefully the right
+               # file will be there. If not, the client will receive 404 from there.
+               url = self.backend.path_to_url(path, mirrored=False)
+
+               self.redirect(url)