hub: Increase robustness if the server goes away

This patch will retry a couple of requests on certain errors.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/pakfire/hub.py b/src/pakfire/hub.py
index 00df9401b1bd145d45fbe50e3133c7b53e3de137..99e2ca06baeda3afd008dc6c8f38fe7976cd2456 100644 (file)
--- a/src/pakfire/hub.py
+++ b/src/pakfire/hub.py
@@ -58,6 +58,18 @@ class AuthError(Exception):
        pass
 
 
+class TransportError(Exception):
+       pass
+
+
+class TemporaryConnectionError(TransportError):
+       """
+               Raised when there is a temporary connection issue and
+               the request should be tried again.
+       """
+       pass
+
+
 class Hub(object):
        def __init__(self, url, keytab=None):
                self.url = url
@@ -150,6 +162,9 @@ class Hub(object):
                req = tornado.httpclient.HTTPRequest(
                        method=method, url=url, headers=headers, body=body,
 
+                       # Give the server more time to respond
+                       request_timeout=60,
+
                        # Add all the rest
                        body_producer=body_producer,
                )
@@ -164,9 +179,16 @@ class Hub(object):
                        )
 
                # Send the request and wait for a response
-               res = await self.client.fetch(req)
+               try:
+                       res = await self.client.fetch(req)
 
-               # XXX Do we have to catch any errors here?
+               # Catch any HTTP errors
+               except tornado.httpclient.HTTPError as e:
+                       if e.code in (502, 503):
+                               raise TemporaryConnectionError from e
+
+                       # Re-raise anything else
+                       raise e
 
                # Perform mutual authentication
                if authenticate:
@@ -319,14 +341,23 @@ class Hub(object):
                # Compute a digest
                digest = self._compute_digest("blake2b", path)
 
-               # Prepare the file for streaming
-               body_producer = functools.partial(self._stream_file, path, size, p)
+               while True:
+                       # Prepare the file for streaming
+                       body_producer = functools.partial(self._stream_file, path, size, p)
 
-               # Perform upload
-               response = await self._request("PUT", "/api/v1/uploads",
-                       body_producer=body_producer,
-                       filename=filename, size=size, digest=digest
-               )
+                       # Perform upload
+                       try:
+                               response = await self._request("PUT", "/api/v1/uploads",
+                                       body_producer=body_producer,
+                                       filename=filename, size=size, digest=digest
+                               )
+
+                       # On temporary issues, try again after a few seconds
+                       except TemporaryConnectionError as e:
+                               await asycio.sleep(5)
+
+                       else:
+                               break
 
                # Return the upload ID
                return response.get("id")
@@ -678,10 +709,19 @@ class JobControlConnection(HubObject):
                if packages:
                        packages = await self.hub.upload_multi(*packages)
 
-               # Send the request
-               response = await self.hub._request("POST", "/api/v1/jobs/%s/finished" % self.id,
-                       success="1" if success else "0", logfile=logfile, packages=packages,
-               )
+               while True:
+                       try:
+                               # Send the request
+                               response = await self.hub._request("POST", "/api/v1/jobs/%s/finished" % self.id,
+                                       success="1" if success else "0", logfile=logfile, packages=packages,
+                               )
+
+                       # Try again after a short moment on connection errors
+                       except TemporaryConnectionError as e:
+                               await asyncio.sleep(5)
+
+                       else:
+                               break
 
                # Handle the response
                # XXX TODO