From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 11 Feb 2025 15:25:21 +0000 (+0000)
Subject: mirrors: Add a universal download load balancer
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=93c873b50a596e2deddb35f55f666d82401be4a8;p=pbs.git

mirrors: Add a universal download load balancer

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/buildservice/mirrors.py b/src/buildservice/mirrors.py
index c2aabcb5..e3b7df06 100644
--- a/src/buildservice/mirrors.py
+++ b/src/buildservice/mirrors.py
@@ -434,6 +434,68 @@ class Mirror(database.Base, database.BackendMixin, database.SoftDeleteMixin):
 		# Run the statement
 		return await self.db.select_one(stmt, "uptime")
 
+	async def serves_file(self, path):
+		"""
+			Checks if this mirror serves the file
+		"""
+		# XXX Skip this if the mirror is not online
+
+		# Make the URL
+		url = self.make_url(path)
+
+		# Make the cache key
+		cache_key = "file-check:%s" % url
+
+		# Check if we have something in the cache
+		serves_file = await self.backend.cache.get(cache_key)
+
+		# Nothing in cache, let's run the check
+		if serves_file is None:
+			serves_file = await self._serves_file(url, cache_key=cache_key)
+
+		return serves_file
+
+	async def _serves_file(self, url, cache_key=None):
+		serves_file = None
+
+		# Send a HEAD request for the URL
+		try:
+			response = await self.backend.httpclient.fetch(
+				url, method="HEAD", follow_redirects=True,
+
+				# Don't allow too much time for the mirror to respond
+				connect_timeout=5, request_timeout=5,
+
+				# Ensure the server responds to all types or requests
+				headers = {
+					"Accept"        : "*/*",
+					"Cache-Control" : "no-cache",
+				}
+			)
+
+		# Catch any HTTP errors
+		except tornado.httpclient.HTTPClientError as e:
+			log.error("Mirror %s returned %s for %s" % (self, e.code, url))
+			serves_file = False
+
+		# If there was no error, we assume this file can be downloaded
+		else:
+			log.debug("Mirror %s seems to be serving %s")
+			serves_file = True
+
+		# Store positive responses in the cache for 24 hours
+		# and negative responses for six hours.
+		if cache_key:
+			if serves_file:
+				ttl = 86400 # 24h
+			else:
+				ttl = 21600 # 6h
+
+			# Store in cache
+			await self.backend.cache.set(cache_key, serves_file, ttl)
+
+		return serves_file
+
 
 class MirrorCheck(database.Base):
 	"""
diff --git a/src/web/__init__.py b/src/web/__init__.py
index e5a07af7..91f040db 100644
--- a/src/web/__init__.py
+++ b/src/web/__init__.py
@@ -169,6 +169,9 @@ class Application(tornado.web.Application):
 			(r"/distros/([A-Za-z0-9\-\.]+)/releases/([\w\-_]+)/edit", distributions.ReleasesEditHandler),
 			(r"/distros/([A-Za-z0-9\-\.]+)/releases/([\w\-_]+)/publish", distributions.ReleasesPublishHandler),
 
+			# Downloads
+			(r"/downloads/(.*)", mirrors.DownloadsHandler),
+
 			# Mirrors
 			(r"/mirrors",					 mirrors.IndexHandler),
 			(r"/mirrors/create",             mirrors.CreateHandler),
diff --git a/src/web/mirrors.py b/src/web/mirrors.py
index f329537a..2ec52028 100644
--- a/src/web/mirrors.py
+++ b/src/web/mirrors.py
@@ -1,9 +1,13 @@
 #!/usr/bin/python
 
+import logging
 import tornado.web
 
 from . import base
 
+# Setup logging
+log = logging.getLogger("pbs.web.mirrors")
+
 class IndexHandler(base.BaseHandler):
 	async def get(self):
 		await self.render("mirrors/index.html", mirrors=self.backend.mirrors)
@@ -117,3 +121,40 @@ class DeleteHandler(base.BaseHandler):
 
 		# Redirect back to all mirrors
 		self.redirect("/mirrors")
+
+
+class DownloadsHandler(base.BaseHandler):
+	"""
+		A universal download redirector which does not need to keep any state.
+
+		It will check if the mirror serves the file and redirect the client. We are
+		starting with the closest mirror and walk through all of the until we have
+		found the right file.
+
+		As a last resort, we will try to serve the file locally.
+	"""
+	async def get(self, path):
+		# Fetch all mirrors for this client
+		mirrors = await self.backend.mirrors.get_mirrors_for_address(self.current_address)
+
+		# Walk through all mirrors
+		for mirror in mirrors:
+			# Skip the mirror if it does not serve the file we are looking for
+			if not await mirror.serves_file(path):
+				continue
+
+			# Log action
+			log.info("Sending %s to download %s from %s", self.current_address, path, mirror)
+
+			# Make the URL
+			url = mirror.make_url(path)
+
+			# Redirect the user
+			return self.redirect(url)
+
+		# If we go here, we did not find any working mirror.
+		# We will send the user to our local file store and hopefully the right
+		# file will be there. If not, the client will receive 404 from there.
+		url = self.backend.path_to_url(path, mirrored=False)
+
+		self.redirect(url)