machine: Add retries for ssh

We've been seeing quite a bit of "connection refused" errors in CI.
These are likely happening because sshd hasn't finished starting
yet.

The proper fix for this is to add notify socket support for systemd
running qemu VMs via virtio sockets, but even if that's added, it
will be a very long time before we can rely on it.

For now, let's add a retry mechanism for SSH connections to make
our CI setup more reliable.

diff --git a/mkosi/machine.py b/mkosi/machine.py
index 956cc03d3f23726c40375ce3156b9cf76fb760ea..3133f8b33a1f6658544eb347e515b772dc4c6c8b 100644 (file)
--- a/mkosi/machine.py
+++ b/mkosi/machine.py
@@ -7,6 +7,7 @@ import os
 import signal
 import subprocess
 import sys
+import time
 import unittest
 from textwrap import dedent
 from typing import Any, Iterator, Optional, Sequence, TextIO, Union
@@ -170,7 +171,20 @@ class Machine:
         else:
             cmdline = run_shell_cmdline(self.args, pipe=True, commands=commands)
 
-        return run(cmdline, check=check, stdout=stdout, stderr=stderr, text=True, timeout=timeout)
+        # The retry logic only applies when running commands against a VM.
+
+        for _ in range(0, 30):
+            try:
+                return run(cmdline, check=check, stdout=stdout, stderr=stderr, text=True, timeout=timeout)
+            except subprocess.CalledProcessError as e:
+                # Return code 255 is used for connection errors by ssh.
+                if self.args.verb != Verb.qemu or e.returncode != 255:
+                    raise
+
+                time.sleep(1)
+
+        die("Failed to establish SSH connection")
+
 
     def kill(self) -> None:
         self.__exit__(None, None, None)