From: Michael Tremer <michael.tremer@ipfire.org>
Date: Mon, 24 Jul 2023 17:02:46 +0000 (+0000)
Subject: hub: Increase robustness if the server goes away
X-Git-Tag: 0.9.29~84
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=1dd799f31d7e8e237c6e3cfb1fd06c6dccdbbddf;p=pakfire.git

hub: Increase robustness if the server goes away

This patch will retry a couple of requests on certain errors.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/pakfire/hub.py b/src/pakfire/hub.py
index 00df9401b..99e2ca06b 100644
--- a/src/pakfire/hub.py
+++ b/src/pakfire/hub.py
@@ -58,6 +58,18 @@ class AuthError(Exception):
 	pass
 
 
+class TransportError(Exception):
+	pass
+
+
+class TemporaryConnectionError(TransportError):
+	"""
+		Raised when there is a temporary connection issue and
+		the request should be tried again.
+	"""
+	pass
+
+
 class Hub(object):
 	def __init__(self, url, keytab=None):
 		self.url = url
@@ -150,6 +162,9 @@ class Hub(object):
 		req = tornado.httpclient.HTTPRequest(
 			method=method, url=url, headers=headers, body=body,
 
+			# Give the server more time to respond
+			request_timeout=60,
+
 			# Add all the rest
 			body_producer=body_producer,
 		)
@@ -164,9 +179,16 @@ class Hub(object):
 			)
 
 		# Send the request and wait for a response
-		res = await self.client.fetch(req)
+		try:
+			res = await self.client.fetch(req)
 
-		# XXX Do we have to catch any errors here?
+		# Catch any HTTP errors
+		except tornado.httpclient.HTTPError as e:
+			if e.code in (502, 503):
+				raise TemporaryConnectionError from e
+
+			# Re-raise anything else
+			raise e
 
 		# Perform mutual authentication
 		if authenticate:
@@ -319,14 +341,23 @@ class Hub(object):
 		# Compute a digest
 		digest = self._compute_digest("blake2b", path)
 
-		# Prepare the file for streaming
-		body_producer = functools.partial(self._stream_file, path, size, p)
+		while True:
+			# Prepare the file for streaming
+			body_producer = functools.partial(self._stream_file, path, size, p)
 
-		# Perform upload
-		response = await self._request("PUT", "/api/v1/uploads",
-			body_producer=body_producer,
-			filename=filename, size=size, digest=digest
-		)
+			# Perform upload
+			try:
+				response = await self._request("PUT", "/api/v1/uploads",
+					body_producer=body_producer,
+					filename=filename, size=size, digest=digest
+				)
+
+			# On temporary issues, try again after a few seconds
+			except TemporaryConnectionError as e:
+				await asycio.sleep(5)
+
+			else:
+				break
 
 		# Return the upload ID
 		return response.get("id")
@@ -678,10 +709,19 @@ class JobControlConnection(HubObject):
 		if packages:
 			packages = await self.hub.upload_multi(*packages)
 
-		# Send the request
-		response = await self.hub._request("POST", "/api/v1/jobs/%s/finished" % self.id,
-			success="1" if success else "0", logfile=logfile, packages=packages,
-		)
+		while True:
+			try:
+				# Send the request
+				response = await self.hub._request("POST", "/api/v1/jobs/%s/finished" % self.id,
+					success="1" if success else "0", logfile=logfile, packages=packages,
+				)
+
+			# Try again after a short moment on connection errors
+			except TemporaryConnectionError as e:
+				await asyncio.sleep(5)
+
+			else:
+				break
 
 		# Handle the response
 		# XXX TODO