[#3840] Adapt Hammer to the /etc/kea-api-password change in kea-ctrl-agent

diff --git a/hammer.py b/hammer.py
index 417b6b178c71d334abcfc2019ad1f13dfdb6e6f4..880f0913ba70ca898c2a0827bec78772da595257 100755 (executable)
--- a/hammer.py
+++ b/hammer.py
@@ -2247,15 +2247,32 @@ def _build_binaries_and_run_ut(system, revision, features, tarball_path, env, ch
                 execute('kea-admin db-init pgsql -u keauser -p keapass -n keadb', dry_run=dry_run)
 
 
-def _check_installed_rpm_or_debs(services_list):
+def _check_installed_rpm_or_debs(services_list, log_text, expect_success_on_start=True):
+    """Check that services log the given text after stopping and starting.
+
+    :param services_list: services to check
+    :type services_list: list
+    :param log_text: text to search
+    :type log_text: str
+    :param expect_success_on_start: whether to expect success on "systemctl start", otherwise expect error.
+                                    Useful for negative checks.
+    :type expect_success_on_start: bool
+    """
     for svc in services_list:
-        execute('sudo systemctl stop %s' % svc)
+        execute(f'sudo systemctl stop {svc}')
         now = datetime.datetime.now()
         timestamp = now.strftime('%Y-%m-%d%H:%M:%S')
-        execute('sudo systemctl start %s' % svc)
-        time.sleep(3)
-        cmd = "sudo journalctl --since %s -u %s | grep '_STARTED Kea'" % (timestamp, svc)
-        execute(cmd, attempts=10, sleep_time_after_attempt=1)
+        rc, _ = execute(f'sudo systemctl start {svc}', capture=True, raise_error=expect_success_on_start)
+        assert rc == 0 if expect_success_on_start else 2
+        logs = ''
+        for _ in range(10):
+            _, logs = execute(f'sudo journalctl --since {timestamp} -u {svc}', capture=True)
+            if log_text in logs:
+                break
+            time.sleep(1)
+        if log_text not in logs:
+            print(logs)
+            raise UnexpectedError(f'{log_text} not in logs')
 
 
 def _build_rpm(system, revision, features, tarball_path, env, check_times, dry_run,
@@ -2311,9 +2328,29 @@ def _build_rpm(system, revision, features, tarball_path, env, check_times, dry_r
         execute('rpm -qa | grep isc-kea | xargs sudo rpm -e', check_times=check_times, dry_run=dry_run, raise_error=False)
         execute(f'sudo rpm -i {rpm_root_path}/RPMS/{arch.strip()}/*rpm', check_times=check_times, dry_run=dry_run)
 
+        # Wait for systemd's rate limit period to pass to avoid "Start request repeated too quickly" after the failed
+        # implicit start from the installation of isc-kea-ctrl-agent above.
+        time.sleep(10)
+        # Reset systemd's rate limit period. Redundant, but just to be safe.
+        execute('sudo systemctl reset-failed kea-ctrl-agent.service', raise_error=False)
+
+        _check_installed_rpm_or_debs(
+            ['kea-ctrl-agent.service'],
+            "Expected a file at path '/etc/kea/kea-api-password'",
+            expect_success_on_start=False,
+        )
+
+        # Wait for systemd's rate limit period to pass to avoid "Start request repeated too quickly" after the failed
+        # implicit start from the installation of isc-kea-ctrl-agent above.
+        time.sleep(10)
+        # Reset systemd's rate limit period. Redundant, but just to be safe.
+        execute('sudo systemctl reset-failed kea-ctrl-agent.service', raise_error=False)
+
+        execute('sudo touch /etc/kea/kea-api-password')
+
         # check if kea services can be started
         services_list = ['kea-dhcp4.service', 'kea-dhcp6.service', 'kea-dhcp-ddns.service', 'kea-ctrl-agent.service']
-        _check_installed_rpm_or_debs(services_list)
+        _check_installed_rpm_or_debs(services_list, '_STARTED Kea')
 
     execute(f'mv {rpm_root_path}/RPMS/{arch.strip()}/*rpm pkgs', check_times=check_times, dry_run=dry_run)
 
@@ -2375,9 +2412,34 @@ def _build_deb(system, revision, features, tarball_path, env, check_times, dry_r
     if 'install' in features:
         # install packages
         execute('sudo dpkg -i kea-src/*deb', check_times=check_times, dry_run=dry_run)
+
+        # Wait for systemd's rate limit period to pass to avoid "Start request repeated too quickly" after the failed
+        # implicit start from the installation of isc-kea-ctrl-agent above.
+        time.sleep(10)
+        # Reset systemd's rate limit period. Redundant, but just to be safe.
+        execute('sudo systemctl reset-failed isc-kea-ctrl-agent.service')
+
+        _check_installed_rpm_or_debs(
+            ['isc-kea-ctrl-agent.service'],
+            "Expected a file at path '/etc/kea/kea-api-password'",
+            expect_success_on_start=False,
+        )
+
+        # Wait for systemd's rate limit period to pass to avoid "Start request repeated too quickly" after the failed
+        # explicit start above.
+        time.sleep(10)
+        # Reset systemd's rate limit period. Redundant, but just to be safe.
+        execute('sudo systemctl reset-failed isc-kea-ctrl-agent.service')
+
+        execute('sudo touch /etc/kea/kea-api-password')
         # check if kea services can be started
-        services_list = ['isc-kea-dhcp4-server.service', 'isc-kea-dhcp6-server.service', 'isc-kea-dhcp-ddns-server.service', 'isc-kea-ctrl-agent.service']
-        _check_installed_rpm_or_debs(services_list)
+        services_list = [
+            'isc-kea-dhcp4-server.service',
+            'isc-kea-dhcp6-server.service',
+            'isc-kea-dhcp-ddns-server.service',
+            'isc-kea-ctrl-agent.service',
+        ]
+        _check_installed_rpm_or_debs(services_list, '_STARTED Kea')
 
 
 def _build_alpine_apk(system, revision, features, tarball_path, env, check_times, dry_run,
@@ -2416,6 +2478,14 @@ def _build_alpine_apk(system, revision, features, tarball_path, env, check_times
         # install packages
         execute('sudo apk add *.apk', cwd='kea-pkg', check_times=check_times, dry_run=dry_run)
 
+
+        exitcode, _ = execute('sudo rc-service kea-ctrl-agent start', capture=True, raise_error=False)
+        assert exitcode == 1
+        _, logs = execute('sudo cat /var/log/kea/kea-ctrl-agent.log', capture=True)
+        assert "Expected a file at path '/etc/kea/kea-api-password'" in logs
+
+        execute('sudo touch /etc/kea/kea-api-password')
+
         # check if kea services can be started
         for svc in ['kea-dhcp4', 'kea-dhcp6', 'kea-ctrl-agent', 'kea-dhcp-ddns']:
             execute('sudo rc-service %s start' % svc)