emergency-action: sleep 5s before rebooting in various cases

This adds a new EMERGENCY_ACTION_SLEEP_5S flag, which when set will
delay the emergency action for 5s. This is supposed to be used together
with EMERGENCY_ACTION_WARN so that users can actually read the message
we output.

We enable this with all emergency action requests that already set
EMERGENCY_ACTION_WARN, except for the 7x ctrl-alt-del burst reboot,
where the user knows what they do and there's no real reason to wait,
they don't need to be informed.

This also enables both EMERGENCY_ACTION_WARN + EMERGENCY_ACTION_SLEEP_5S
for FailureAction= processing of regular units, where these were so far
off. (it leaves this off for SuccessAction= however!). This is a good
thing to make things more debuggable: if something fails and we reboot
this really deserves notification of the user.

(For SuccessAction= this logic does not apply, since the shutdown action
induced here is apparently intended part of the codeflow, for example in
systemd-reboot.service or a similar unit, where the shutdown is goal and
not exception and derserves no additional noisy reporting).

Inspired by: https://github.com/systemd/systemd/pull/36705#issuecomment-2717014120

diff --git a/src/core/emergency-action.c b/src/core/emergency-action.c
index 75c26df52f02e27b517b1feffcf9f1a8b2cf24ab..2d756b306c9f64eb63ecf55c089ed119a098489b 100644 (file)
--- a/src/core/emergency-action.c
+++ b/src/core/emergency-action.c
@@ -32,18 +32,47 @@ static const char* const emergency_action_table[_EMERGENCY_ACTION_MAX] = {
         [EMERGENCY_ACTION_HALT_IMMEDIATE]     = "halt-immediate",
 };
 
-static void log_and_status(Manager *m, bool warn, const char *message, const char *reason) {
-        log_full(warn ? LOG_WARNING : LOG_DEBUG, "%s: %s", message, reason);
-        if (warn)
-                manager_status_printf(m, STATUS_TYPE_EMERGENCY,
-                                      ANSI_HIGHLIGHT_RED "  !!  " ANSI_NORMAL,
-                                      "%s: %s", message, reason);
+static void log_and_status(
+                Manager *m,
+                EmergencyAction action,
+                EmergencyActionFlags flags,
+                const char *message,
+                const char *reason) {
+
+        assert(m);
+        assert(message);
+        assert(reason);
+
+        log_full(FLAGS_SET(flags, EMERGENCY_ACTION_WARN) ? LOG_WARNING : LOG_DEBUG,
+                 "%s: %s", message, reason);
+
+        bool do_sleep = FLAGS_SET(flags, EMERGENCY_ACTION_WARN|EMERGENCY_ACTION_SLEEP_5S) &&
+                IN_SET(action,
+                       EMERGENCY_ACTION_EXIT_FORCE,
+                       EMERGENCY_ACTION_REBOOT_FORCE, EMERGENCY_ACTION_REBOOT_IMMEDIATE,
+                       EMERGENCY_ACTION_POWEROFF_FORCE, EMERGENCY_ACTION_POWEROFF_IMMEDIATE,
+                       EMERGENCY_ACTION_SOFT_REBOOT_FORCE,
+                       EMERGENCY_ACTION_KEXEC_FORCE);
+
+        if (FLAGS_SET(flags, EMERGENCY_ACTION_WARN))
+                manager_status_printf(
+                                m,
+                                STATUS_TYPE_EMERGENCY,
+                                ANSI_HIGHLIGHT_RED "  !!  " ANSI_NORMAL,
+                                "%s: %s%s", message, reason,
+                                do_sleep ? ", proceeding in 5s" : "");
+
+        /* Optionally sleep for 5s so that the user can see this output, before we actually execute the
+         * operation. Do this only if we immediately execute an operation, i.e. when there's no event loop to
+         * feed anymore. */
+        if (do_sleep)
+                (void) sleep(5);
 }
 
 void emergency_action(
                 Manager *m,
                 EmergencyAction action,
-                EmergencyActionFlags options,
+                EmergencyActionFlags flags,
                 const char *reboot_arg,
                 int exit_status,
                 const char *reason) {
@@ -53,6 +82,11 @@ void emergency_action(
         assert(m);
         assert(action >= 0);
         assert(action < _EMERGENCY_ACTION_MAX);
+        assert((flags & ~_EMERGENCY_ACTION_FLAGS_MAX) == 0);
+        assert(reason);
+
+        if (action == EMERGENCY_ACTION_NONE)
+                return;
 
         /* Is the special shutdown target active or queued? If so, we are in shutdown state */
         if (IN_SET(action,
@@ -70,34 +104,29 @@ void emergency_action(
                 }
         }
 
-        if (action == EMERGENCY_ACTION_NONE)
-                return;
-
-        if (FLAGS_SET(options, EMERGENCY_ACTION_IS_WATCHDOG) && !m->service_watchdogs) {
+        if (FLAGS_SET(flags, EMERGENCY_ACTION_IS_WATCHDOG) && !m->service_watchdogs) {
                 log_warning("Watchdog disabled! Not acting on: %s", reason);
                 return;
         }
 
-        bool warn = FLAGS_SET(options, EMERGENCY_ACTION_WARN);
-
         switch (action) {
 
         case EMERGENCY_ACTION_REBOOT:
-                log_and_status(m, warn, "Rebooting", reason);
+                log_and_status(m, action, flags, "Rebooting", reason);
 
                 (void) update_reboot_parameter_and_warn(reboot_arg, true);
                 (void) manager_add_job_by_name_and_warn(m, JOB_START, SPECIAL_REBOOT_TARGET, JOB_REPLACE_IRREVERSIBLY, NULL, NULL);
                 break;
 
         case EMERGENCY_ACTION_REBOOT_FORCE:
-                log_and_status(m, warn, "Forcibly rebooting", reason);
+                log_and_status(m, action, flags, "Forcibly rebooting", reason);
 
                 (void) update_reboot_parameter_and_warn(reboot_arg, true);
                 m->objective = MANAGER_REBOOT;
                 break;
 
         case EMERGENCY_ACTION_REBOOT_IMMEDIATE:
-                log_and_status(m, warn, "Rebooting immediately", reason);
+                log_and_status(m, action, flags, "Rebooting immediately", reason);
 
                 sync();
 
@@ -112,13 +141,13 @@ void emergency_action(
                 break;
 
         case EMERGENCY_ACTION_SOFT_REBOOT:
-                log_and_status(m, warn, "Soft-rebooting", reason);
+                log_and_status(m, action, flags, "Soft-rebooting", reason);
 
                 (void) manager_add_job_by_name_and_warn(m, JOB_START, SPECIAL_SOFT_REBOOT_TARGET, JOB_REPLACE_IRREVERSIBLY, NULL, NULL);
                 break;
 
         case EMERGENCY_ACTION_SOFT_REBOOT_FORCE:
-                log_and_status(m, warn, "Forcibly soft-rebooting", reason);
+                log_and_status(m, action, flags, "Forcibly soft-rebooting", reason);
 
                 m->objective = MANAGER_SOFT_REBOOT;
                 break;
@@ -129,7 +158,7 @@ void emergency_action(
                         m->return_value = exit_status;
 
                 if (MANAGER_IS_USER(m) || detect_container() > 0) {
-                        log_and_status(m, warn, "Exiting", reason);
+                        log_and_status(m, action, flags, "Exiting", reason);
                         (void) manager_add_job_by_name_and_warn(m, JOB_START, SPECIAL_EXIT_TARGET, JOB_REPLACE_IRREVERSIBLY, NULL, NULL);
                         break;
                 }
@@ -138,7 +167,7 @@ void emergency_action(
                 _fallthrough_;
 
         case EMERGENCY_ACTION_POWEROFF:
-                log_and_status(m, warn, "Powering off", reason);
+                log_and_status(m, action, flags, "Powering off", reason);
                 (void) manager_add_job_by_name_and_warn(m, JOB_START, SPECIAL_POWEROFF_TARGET, JOB_REPLACE_IRREVERSIBLY, NULL, NULL);
                 break;
 
@@ -148,7 +177,7 @@ void emergency_action(
                         m->return_value = exit_status;
 
                 if (MANAGER_IS_USER(m) || detect_container() > 0) {
-                        log_and_status(m, warn, "Exiting immediately", reason);
+                        log_and_status(m, action, flags, "Exiting immediately", reason);
                         m->objective = MANAGER_EXIT;
                         break;
                 }
@@ -157,12 +186,12 @@ void emergency_action(
                 _fallthrough_;
 
         case EMERGENCY_ACTION_POWEROFF_FORCE:
-                log_and_status(m, warn, "Forcibly powering off", reason);
+                log_and_status(m, action, flags, "Forcibly powering off", reason);
                 m->objective = MANAGER_POWEROFF;
                 break;
 
         case EMERGENCY_ACTION_POWEROFF_IMMEDIATE:
-                log_and_status(m, warn, "Powering off immediately", reason);
+                log_and_status(m, action, flags, "Powering off immediately", reason);
 
                 sync();
 
@@ -171,27 +200,27 @@ void emergency_action(
                 break;
 
         case EMERGENCY_ACTION_KEXEC:
-                log_and_status(m, warn, "Executing kexec", reason);
+                log_and_status(m, action, flags, "Executing kexec", reason);
                 (void) manager_add_job_by_name_and_warn(m, JOB_START, SPECIAL_KEXEC_TARGET, JOB_REPLACE_IRREVERSIBLY, NULL, NULL);
                 break;
 
         case EMERGENCY_ACTION_KEXEC_FORCE:
-                log_and_status(m, warn, "Forcibly executing kexec", reason);
+                log_and_status(m, action, flags, "Forcibly executing kexec", reason);
                 m->objective = MANAGER_KEXEC;
                 break;
 
         case EMERGENCY_ACTION_HALT:
-                log_and_status(m, warn, "Halting", reason);
+                log_and_status(m, action, flags, "Halting", reason);
                 (void) manager_add_job_by_name_and_warn(m, JOB_START, SPECIAL_HALT_TARGET, JOB_REPLACE_IRREVERSIBLY, NULL, NULL);
                 break;
 
         case EMERGENCY_ACTION_HALT_FORCE:
-                log_and_status(m, warn, "Forcibly halting", reason);
+                log_and_status(m, action, flags, "Forcibly halting", reason);
                 m->objective = MANAGER_HALT;
                 break;
 
         case EMERGENCY_ACTION_HALT_IMMEDIATE:
-                log_and_status(m, warn, "Halting immediately", reason);
+                log_and_status(m, action, flags, "Halting immediately", reason);
 
                 sync();
 

diff --git a/src/core/emergency-action.h b/src/core/emergency-action.h
index 6bec47511d69e9ff38c780869137c4490959d0a3..5f7b0b4bc9bd70818515457468c9d91fe1eb9848 100644 (file)
--- a/src/core/emergency-action.h
+++ b/src/core/emergency-action.h
@@ -28,16 +28,22 @@ typedef enum EmergencyAction {
 } EmergencyAction;
 
 typedef enum EmergencyActionFlags {
-        EMERGENCY_ACTION_IS_WATCHDOG = 1 << 0,
-        EMERGENCY_ACTION_WARN        = 1 << 1,
+        EMERGENCY_ACTION_IS_WATCHDOG = 1 << 0, /* this action triggered by a watchdog or other kind of timeout */
+        EMERGENCY_ACTION_WARN        = 1 << 1, /* log at LOG_WARNING + write to system console */
+        EMERGENCY_ACTION_SLEEP_5S    = 1 << 2, /* wait 5s before executing action; only honoured together with EMERGENCY_ACTION_WARN */
+        _EMERGENCY_ACTION_FLAGS_MAX  = (1 << 3) - 1,
 } EmergencyActionFlags;
 
 #include "macro.h"
 #include "manager.h"
 
-void emergency_action(Manager *m,
-                      EmergencyAction action, EmergencyActionFlags options,
-                      const char *reboot_arg, int exit_status, const char *reason);
+void emergency_action(
+                Manager *m,
+                EmergencyAction action,
+                EmergencyActionFlags flags,
+                const char *reboot_arg,
+                int exit_status,
+                const char *reason);
 
 const char* emergency_action_to_string(EmergencyAction i) _const_;
 EmergencyAction emergency_action_from_string(const char *s) _pure_;

diff --git a/src/core/job.c b/src/core/job.c
index f674a7845f8d46f966773c2a6e06be8b68c02a5b..200bebcf8a57ed1355aab7ed1126036233ba82c2 100644 (file)
--- a/src/core/job.c
+++ b/src/core/job.c
@@ -1123,9 +1123,13 @@ static int job_dispatch_timer(sd_event_source *s, uint64_t monotonic, void *user
         u = j->unit;
         job_finish_and_invalidate(j, JOB_TIMEOUT, true, false);
 
-        emergency_action(u->manager, u->job_timeout_action,
-                         EMERGENCY_ACTION_IS_WATCHDOG|EMERGENCY_ACTION_WARN,
-                         u->job_timeout_reboot_arg, -1, "job timed out");
+        emergency_action(
+                        u->manager,
+                        u->job_timeout_action,
+                        EMERGENCY_ACTION_IS_WATCHDOG|EMERGENCY_ACTION_WARN|EMERGENCY_ACTION_SLEEP_5S,
+                        u->job_timeout_reboot_arg,
+                        /* exit_status= */ -1,
+                        "job timed out");
 
         return 0;
 }

diff --git a/src/core/manager.c b/src/core/manager.c
index 36217ead14a489196ce5b3a5dadbc5de49f8b744..6a4fd3e991adcede669681e6cf4f96ae97d11f12 100644 (file)
--- a/src/core/manager.c
+++ b/src/core/manager.c
@@ -2976,8 +2976,13 @@ static void manager_handle_ctrl_alt_del(Manager *m) {
         if (ratelimit_below(&m->ctrl_alt_del_ratelimit) || m->cad_burst_action == EMERGENCY_ACTION_NONE)
                 manager_start_special(m, SPECIAL_CTRL_ALT_DEL_TARGET, JOB_REPLACE_IRREVERSIBLY);
         else
-                emergency_action(m, m->cad_burst_action, EMERGENCY_ACTION_WARN, NULL, -1,
-                                 "Ctrl-Alt-Del was pressed more than 7 times within 2s");
+                emergency_action(
+                                m,
+                                m->cad_burst_action,
+                                EMERGENCY_ACTION_WARN,
+                                /* reboot_arg= */ NULL,
+                                /* exit_status= */ -1,
+                                "Ctrl-Alt-Del was pressed more than 7 times within 2s");
 }
 
 static int manager_dispatch_signal_fd(sd_event_source *source, int fd, uint32_t revents, void *userdata) {

diff --git a/src/core/unit.c b/src/core/unit.c
index 781de8d594391c15e2eea22462fe8343d044a394..ee45faebb61363a1ba91c02f2db89ea61c60956e 100644 (file)
--- a/src/core/unit.c
+++ b/src/core/unit.c
@@ -1821,9 +1821,13 @@ int unit_test_start_limit(Unit *u) {
 
         reason = strjoina("unit ", u->id, " failed");
 
-        emergency_action(u->manager, u->start_limit_action,
-                         EMERGENCY_ACTION_IS_WATCHDOG|EMERGENCY_ACTION_WARN,
-                         u->reboot_arg, -1, reason);
+        emergency_action(
+                        u->manager,
+                        u->start_limit_action,
+                        EMERGENCY_ACTION_IS_WATCHDOG|EMERGENCY_ACTION_WARN|EMERGENCY_ACTION_SLEEP_5S,
+                        u->reboot_arg,
+                        /* exit_status= */ -1,
+                        reason);
 
         return -ECANCELED;
 }
@@ -2710,10 +2714,10 @@ void unit_notify(Unit *u, UnitActiveState os, UnitActiveState ns, bool reload_su
 
                 if (os != UNIT_FAILED && ns == UNIT_FAILED) {
                         reason = strjoina("unit ", u->id, " failed");
-                        emergency_action(m, u->failure_action, 0, u->reboot_arg, unit_failure_action_exit_status(u), reason);
+                        emergency_action(m, u->failure_action, EMERGENCY_ACTION_WARN|EMERGENCY_ACTION_SLEEP_5S, u->reboot_arg, unit_failure_action_exit_status(u), reason);
                 } else if (!UNIT_IS_INACTIVE_OR_FAILED(os) && ns == UNIT_INACTIVE) {
                         reason = strjoina("unit ", u->id, " succeeded");
-                        emergency_action(m, u->success_action, 0, u->reboot_arg, unit_success_action_exit_status(u), reason);
+                        emergency_action(m, u->success_action, /* flags= */ 0, u->reboot_arg, unit_success_action_exit_status(u), reason);
                 }
         }