ftests: Retry run() command after websocket failure

LXC/LXD will sometimes throw an "Error: websocket: bad handshake" error
on underpowered machines.  Wait a bit (arbitrarily chosen 5 seconds) and
retry the command.

Signed-off-by: Tom Hromatka <tom.hromatka@oracle.com>
Reviewed-by: Kamalesh Babulal <kamalesh.babulal@oracle.com>

diff --git a/tests/ftests/run.py b/tests/ftests/run.py
index b6bd3824920403bab609e6e99e682d994ecd3040..c985162d00df1723ac5b0fda093c202337dd5652 100644 (file)
--- a/tests/ftests/run.py
+++ b/tests/ftests/run.py
@@ -9,11 +9,12 @@
 from subprocess import TimeoutExpired
 from log import Log
 import subprocess
+import time
 
 
 class Run(object):
     @staticmethod
-    def run(command, shell_bool=False, ignore_profiling_errors=True, timeout=None):
+    def __run(command, shell_bool, timeout):
         if shell_bool:
             if isinstance(command, str):
                 # nothing to do.  command is already formatted as a string
@@ -68,6 +69,19 @@ class Run(object):
                             ''.format(' '.join(command), ret, out, err)
                          )
 
+        return ret, out, err
+
+    @staticmethod
+    def run(command, shell_bool=False, ignore_profiling_errors=True, timeout=None):
+        ret, out, err = Run.__run(command, shell_bool, timeout)
+
+        if err.find('Error: websocket: bad handshake') >= 0:
+            # LXD sometimes throws this error on underpowered machines.
+            # Wait a bit, then try the request again
+            Log.log_error('Received \'{}\' error.  Re-running command'.format(err))
+            time.sleep(5)
+            ret, out, err = Run.__run(command, shell_bool, timeout)
+
         if ret != 0:
             raise RunError("Command '{}' failed".format(''.join(command)),
                            command, ret, out, err)