src/shared/watchdog.c

   1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
   2
   3 #include <errno.h>
   4 #include <fcntl.h>
   5 #include <sys/ioctl.h>
   6 #include <syslog.h>
   7 #include <unistd.h>
   8 #include <linux/watchdog.h>
   9
  10 #include "devnum-util.h"
  11 #include "errno-util.h"
  12 #include "fd-util.h"
  13 #include "fileio.h"
  14 #include "log.h"
  15 #include "path-util.h"
  16 #include "string-util.h"
  17 #include "time-util.h"
  18 #include "watchdog.h"
  19
  20 static int watchdog_fd = -EBADF;
  21 static char *watchdog_device = NULL;
  22 static usec_t watchdog_timeout = 0; /* 0 → close device and USEC_INFINITY → don't change timeout */
  23 static usec_t watchdog_pretimeout = 0; /* 0 → disable pretimeout and USEC_INFINITY → don't change pretimeout */
  24 static usec_t watchdog_last_ping = USEC_INFINITY;
  25 static bool watchdog_supports_pretimeout = false; /* Depends on kernel state that might change at runtime */
  26 static char *watchdog_pretimeout_governor = NULL;
  27
  28 /* Starting from kernel version 4.5, the maximum allowable watchdog timeout is
  29  * UINT_MAX/1000U seconds (since internal calculations are done in milliseconds
  30  * using unsigned integers. However, the kernel's userspace API for the watchdog
  31  * uses signed integers for its ioctl parameters (even for timeout values and
  32  * bit flags) so this is why we must consider the maximum signed integer value
  33  * as well.
  34  */
  35 #define WATCHDOG_TIMEOUT_MAX_SEC (CONST_MIN(UINT_MAX/1000U, (unsigned)INT_MAX))
  36
  37 #define WATCHDOG_GOV_NAME_MAXLEN 20 /* From the kernel watchdog driver */
  38
  39 static int saturated_usec_to_sec(usec_t val) {
  40         usec_t t = DIV_ROUND_UP(val, USEC_PER_SEC);
  41         return MIN(t, (usec_t) WATCHDOG_TIMEOUT_MAX_SEC); /* Saturate to watchdog max */
  42 }
  43
  44 static int get_watchdog_sysfs_path(const char *filename, char **ret_path) {
  45         struct stat st;
  46
  47         if (watchdog_fd < 0)
  48                 return -EBADF;
  49
  50         if (fstat(watchdog_fd, &st))
  51                 return -errno;
  52
  53         if (!S_ISCHR(st.st_mode))
  54                 return -EBADF;
  55
  56         if (asprintf(ret_path, "/sys/dev/char/"DEVNUM_FORMAT_STR"/%s", DEVNUM_FORMAT_VAL(st.st_rdev), filename) < 0)
  57                 return -ENOMEM;
  58
  59         return 0;
  60 }
  61
  62 static int get_pretimeout_governor(char **ret_gov) {
  63         _cleanup_free_ char *sys_fn = NULL;
  64         int r;
  65
  66         r = get_watchdog_sysfs_path("pretimeout_governor", &sys_fn);
  67         if (r < 0)
  68                 return r;
  69
  70         log_info("Watchdog: reading from %s", sys_fn);
  71
  72         r = read_virtual_file(sys_fn, WATCHDOG_GOV_NAME_MAXLEN - 1, ret_gov, NULL);
  73         if (r < 0)
  74                 return r;
  75
  76         delete_trailing_chars(*ret_gov, WHITESPACE);
  77
  78         return 0;
  79 }
  80
  81 static int set_pretimeout_governor(const char *governor) {
  82         _cleanup_free_ char *sys_fn = NULL;
  83         int r;
  84
  85         if (isempty(governor))
  86                 return 0; /* Nothing to do */
  87
  88         r = get_watchdog_sysfs_path("pretimeout_governor", &sys_fn);
  89         if (r < 0)
  90                 return r;
  91
  92         log_info("Watchdog: setting pretimeout_governor to '%s' via '%s'", governor, sys_fn);
  93
  94         r = write_string_file(sys_fn,
  95                               governor,
  96                               WRITE_STRING_FILE_DISABLE_BUFFER | WRITE_STRING_FILE_VERIFY_ON_FAILURE | WRITE_STRING_FILE_VERIFY_IGNORE_NEWLINE);
  97         if (r < 0)
  98                 return log_error_errno(r, "Failed to set watchdog pretimeout_governor to '%s': %m", governor);
  99
 100         return r;
 101 }
 102
 103 static int watchdog_set_enable(bool enable) {
 104         int flags = enable ? WDIOS_ENABLECARD : WDIOS_DISABLECARD;
 105
 106         assert(watchdog_fd >= 0);
 107
 108         if (ioctl(watchdog_fd, WDIOC_SETOPTIONS, &flags) < 0) {
 109                 if (!enable)
 110                         return log_warning_errno(errno, "Failed to disable hardware watchdog, ignoring: %m");
 111
 112                 /* ENOTTY means the watchdog is always enabled so we're fine */
 113                 log_full_errno(ERRNO_IS_NOT_SUPPORTED(errno) ? LOG_DEBUG : LOG_WARNING, errno,
 114                                "Failed to enable hardware watchdog, ignoring: %m");
 115                 if (!ERRNO_IS_NOT_SUPPORTED(errno))
 116                         return -errno;
 117         }
 118
 119         return 0;
 120 }
 121
 122 static int watchdog_read_timeout(void) {
 123         int sec = 0;
 124
 125         assert(watchdog_fd >= 0);
 126
 127         if (ioctl(watchdog_fd, WDIOC_GETTIMEOUT, &sec) < 0)
 128                 return -errno;
 129
 130         assert(sec > 0);
 131         watchdog_timeout = sec * USEC_PER_SEC;
 132
 133         return 0;
 134 }
 135
 136 static int watchdog_set_timeout(void) {
 137         int sec;
 138
 139         assert(watchdog_fd >= 0);
 140         assert(timestamp_is_set(watchdog_timeout));
 141
 142         sec = saturated_usec_to_sec(watchdog_timeout);
 143
 144         if (ioctl(watchdog_fd, WDIOC_SETTIMEOUT, &sec) < 0)
 145                 return -errno;
 146
 147         assert(sec > 0); /* buggy driver ? */
 148         watchdog_timeout = sec * USEC_PER_SEC;
 149
 150         return 0;
 151 }
 152
 153 static int watchdog_read_pretimeout(void) {
 154         int sec = 0;
 155
 156         assert(watchdog_fd >= 0);
 157
 158         if (ioctl(watchdog_fd, WDIOC_GETPRETIMEOUT, &sec) < 0) {
 159                 watchdog_pretimeout = 0;
 160                 return log_full_errno(ERRNO_IS_NOT_SUPPORTED(errno) ? LOG_DEBUG : LOG_WARNING, errno, "Failed to get watchdog pretimeout value, ignoring: %m");
 161         }
 162
 163         watchdog_pretimeout = sec * USEC_PER_SEC;
 164
 165         return 0;
 166 }
 167
 168 static int watchdog_set_pretimeout(void) {
 169         int sec;
 170
 171         assert(watchdog_fd >= 0);
 172         assert(watchdog_pretimeout != USEC_INFINITY);
 173
 174         sec = saturated_usec_to_sec(watchdog_pretimeout);
 175
 176         if (ioctl(watchdog_fd, WDIOC_SETPRETIMEOUT, &sec) < 0) {
 177                 watchdog_pretimeout = 0;
 178
 179                 if (ERRNO_IS_NOT_SUPPORTED(errno)) {
 180                         log_info("Watchdog does not support pretimeouts.");
 181                         return 0;
 182                 }
 183
 184                 return log_error_errno(errno, "Failed to set watchdog pretimeout to %s: %m", FORMAT_TIMESPAN(sec, USEC_PER_SEC));
 185         }
 186
 187         /* The set ioctl does not return the actual value set so get it now. */
 188         (void) watchdog_read_pretimeout();
 189
 190         return 0;
 191 }
 192
 193 usec_t watchdog_get_last_ping(clockid_t clock) {
 194         return map_clock_usec(watchdog_last_ping, CLOCK_BOOTTIME, clock);
 195 }
 196
 197 static int watchdog_ping_now(void) {
 198         assert(watchdog_fd >= 0);
 199
 200         if (ioctl(watchdog_fd, WDIOC_KEEPALIVE, 0) < 0)
 201                 return log_warning_errno(errno, "Failed to ping hardware watchdog, ignoring: %m");
 202
 203         watchdog_last_ping = now(CLOCK_BOOTTIME);
 204
 205         return 0;
 206 }
 207
 208 static int update_pretimeout(void) {
 209         _cleanup_free_ char *governor = NULL;
 210         int r, t_sec, pt_sec;
 211
 212         if (watchdog_fd < 0)
 213                 return 0;
 214
 215         if (watchdog_timeout == USEC_INFINITY || watchdog_pretimeout == USEC_INFINITY)
 216                 return 0;
 217
 218         if (!watchdog_supports_pretimeout && watchdog_pretimeout == 0)
 219                 return 0; /* Nothing to do */
 220
 221         /* The configuration changed, do not assume it can still work, as the module(s)
 222          * might have been unloaded. */
 223         watchdog_supports_pretimeout = false;
 224
 225         /* Update the pretimeout governor as well */
 226         (void) set_pretimeout_governor(watchdog_pretimeout_governor);
 227
 228         r = get_pretimeout_governor(&governor);
 229         if (r < 0)
 230                 return log_warning_errno(r, "Watchdog: failed to read pretimeout governor: %m");
 231         if (isempty(governor))
 232                 return log_warning_errno(SYNTHETIC_ERRNO(EINVAL),
 233                                          "Watchdog: no pretimeout governor detected - is the required kernel module loaded?");
 234
 235         /* If we have a pretimeout governor, then pretimeout is supported. Without a governor
 236          * pretimeout does not work at all.
 237          * Note that this might require a kernel module that is not autoloaded, so we don't
 238          * cache this, but we check every time the configuration changes. */
 239         watchdog_supports_pretimeout = true;
 240
 241         /* Determine if the pretimeout is valid for the current watchdog timeout. */
 242         t_sec = saturated_usec_to_sec(watchdog_timeout);
 243         pt_sec = saturated_usec_to_sec(watchdog_pretimeout);
 244         if (pt_sec >= t_sec) {
 245                 r = log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 246                                     "Cannot set watchdog pretimeout to %is (%s watchdog timeout of %is)",
 247                                     pt_sec, pt_sec == t_sec ? "same as" : "longer than", t_sec);
 248                 (void) watchdog_read_pretimeout();
 249         } else
 250                 r = watchdog_set_pretimeout();
 251
 252         if (watchdog_pretimeout == 0)
 253                 log_info("Watchdog pretimeout is disabled.");
 254         else
 255                 log_info("Watchdog running with a pretimeout of %s with governor '%s'.",
 256                          FORMAT_TIMESPAN(watchdog_pretimeout, 0),
 257                          governor);
 258
 259         return r;
 260 }
 261
 262 static int update_timeout(void) {
 263         int r;
 264         usec_t previous_timeout;
 265
 266         assert(watchdog_timeout > 0);
 267
 268         if (watchdog_fd < 0)
 269                 return 0;
 270
 271         previous_timeout = watchdog_timeout;
 272
 273         if (watchdog_timeout != USEC_INFINITY) {
 274                 r = watchdog_set_timeout();
 275                 if (r < 0) {
 276                         if (!ERRNO_IS_NOT_SUPPORTED(r))
 277                                 return log_error_errno(r, "Failed to set watchdog hardware timeout to %s: %m",
 278                                                        FORMAT_TIMESPAN(watchdog_timeout, 0));
 279
 280                         log_info("Modifying watchdog hardware timeout is not supported, reusing the programmed timeout.");
 281                         watchdog_timeout = USEC_INFINITY;
 282                 }
 283         }
 284
 285         if (watchdog_timeout == USEC_INFINITY) {
 286                 r = watchdog_read_timeout();
 287                 if (r < 0) {
 288                         if (!ERRNO_IS_NOT_SUPPORTED(r))
 289                                 return log_error_errno(r, "Failed to query watchdog hardware timeout: %m");
 290                         log_info("Reading watchdog hardware timeout is not supported, reusing the configured timeout.");
 291                         watchdog_timeout = previous_timeout;
 292                 }
 293         }
 294
 295         /* If the watchdog timeout was changed, the pretimeout could have been
 296          * changed as well by the driver or the kernel so we need to update the
 297          * pretimeout now. Or if the watchdog is being configured for the first
 298          * time, we want to configure the pretimeout before it is enabled. */
 299         (void) update_pretimeout();
 300
 301         r = watchdog_set_enable(true);
 302         if (r < 0)
 303                 return r;
 304
 305         log_info("Watchdog running with a hardware timeout of %s.", FORMAT_TIMESPAN(watchdog_timeout, 0));
 306
 307         return watchdog_ping_now();
 308 }
 309
 310 static int open_watchdog(void) {
 311         struct watchdog_info ident;
 312         char **try_order;
 313         int r;
 314
 315         if (watchdog_fd >= 0)
 316                 return 0;
 317
 318         /* Let's prefer new-style /dev/watchdog0 (i.e. kernel 3.5+) over classic /dev/watchdog. The former
 319          * has the benefit that we can easily find the matching directory in sysfs from it, as the relevant
 320          * sysfs attributes can only be found via /sys/dev/char/<major>:<minor> if the new-style device
 321          * major/minor is used, not the old-style. */
 322         try_order = !watchdog_device || PATH_IN_SET(watchdog_device, "/dev/watchdog", "/dev/watchdog0") ?
 323                 STRV_MAKE("/dev/watchdog0", "/dev/watchdog") : STRV_MAKE(watchdog_device);
 324
 325         STRV_FOREACH(wd, try_order) {
 326                 watchdog_fd = open(*wd, O_WRONLY|O_CLOEXEC);
 327                 if (watchdog_fd >= 0) {
 328                         if (free_and_strdup(&watchdog_device, *wd) < 0) {
 329                                 r = log_oom_debug();
 330                                 goto close_and_fail;
 331                         }
 332
 333                         break;
 334                 }
 335
 336                 if (errno != ENOENT)
 337                         return log_debug_errno(errno, "Failed to open watchdog device %s: %m", *wd);
 338         }
 339
 340         if (watchdog_fd < 0)
 341                 return log_debug_errno(SYNTHETIC_ERRNO(ENOENT), "Failed to open watchdog device %s.", watchdog_device ?: "auto");
 342
 343         if (ioctl(watchdog_fd, WDIOC_GETSUPPORT, &ident) < 0)
 344                 log_debug_errno(errno, "Hardware watchdog %s does not support WDIOC_GETSUPPORT ioctl, ignoring: %m", watchdog_device);
 345         else
 346                 log_info("Using hardware watchdog '%s', version %x, device %s",
 347                          ident.identity,
 348                          ident.firmware_version,
 349                          watchdog_device);
 350
 351         r = update_timeout();
 352         if (r < 0)
 353                 goto close_and_fail;
 354
 355         return 0;
 356
 357 close_and_fail:
 358         watchdog_close(/* disarm= */ true);
 359         return r;
 360 }
 361
 362 const char *watchdog_get_device(void) {
 363         return watchdog_device;
 364 }
 365
 366 int watchdog_set_device(const char *path) {
 367         int r;
 368
 369         r = free_and_strdup(&watchdog_device, path);
 370         if (r > 0) /* watchdog_device changed */
 371                 watchdog_close(/* disarm= */ true);
 372
 373         return r;
 374 }
 375
 376 int watchdog_setup(usec_t timeout) {
 377         usec_t previous_timeout;
 378         int r;
 379
 380         /* timeout=0 closes the device whereas passing timeout=USEC_INFINITY opens it (if needed)
 381          * without configuring any particular timeout and thus reuses the programmed value (therefore
 382          * it's a nop if the device is already opened). */
 383
 384         if (timeout == 0) {
 385                 watchdog_close(true);
 386                 return 0;
 387         }
 388
 389         /* Let's shortcut duplicated requests */
 390         if (watchdog_fd >= 0 && (timeout == watchdog_timeout || timeout == USEC_INFINITY))
 391                 return 0;
 392
 393         /* Initialize the watchdog timeout with the caller value. This value is going to be updated by
 394          * update_timeout() with the closest value supported by the driver */
 395         previous_timeout = watchdog_timeout;
 396         watchdog_timeout = timeout;
 397
 398         if (watchdog_fd < 0)
 399                 return open_watchdog();
 400
 401         r = update_timeout();
 402         if (r < 0)
 403                 watchdog_timeout = previous_timeout;
 404
 405         return r;
 406 }
 407
 408 int watchdog_setup_pretimeout(usec_t timeout) {
 409         /* timeout=0 disables the pretimeout whereas timeout=USEC_INFINITY is a nop. */
 410         if ((watchdog_fd >= 0 && timeout == watchdog_pretimeout) || timeout == USEC_INFINITY)
 411                 return 0;
 412
 413         /* Initialize the watchdog timeout with the caller value. This value is
 414          * going to be updated by update_pretimeout() with the running value,
 415          * even if it fails to update the timeout. */
 416         watchdog_pretimeout = timeout;
 417
 418         return update_pretimeout();
 419 }
 420
 421 int watchdog_setup_pretimeout_governor(const char *governor) {
 422         if (free_and_strdup(&watchdog_pretimeout_governor, governor) < 0)
 423                 return -ENOMEM;
 424
 425         return set_pretimeout_governor(watchdog_pretimeout_governor);
 426 }
 427
 428 static usec_t calc_timeout(void) {
 429         /* Calculate the effective timeout which accounts for the watchdog
 430          * pretimeout if configured and supported. */
 431         if (watchdog_supports_pretimeout && timestamp_is_set(watchdog_pretimeout) && watchdog_timeout >= watchdog_pretimeout)
 432                 return watchdog_timeout - watchdog_pretimeout;
 433         else
 434                 return watchdog_timeout;
 435 }
 436
 437 usec_t watchdog_runtime_wait(void) {
 438         usec_t timeout = calc_timeout();
 439         if (!timestamp_is_set(timeout))
 440                 return USEC_INFINITY;
 441
 442         /* Sleep half the watchdog timeout since the last successful ping at most */
 443         if (timestamp_is_set(watchdog_last_ping)) {
 444                 usec_t ntime = now(CLOCK_BOOTTIME);
 445
 446                 assert(ntime >= watchdog_last_ping);
 447                 return usec_sub_unsigned(watchdog_last_ping + (timeout / 2), ntime);
 448         }
 449
 450         return timeout / 2;
 451 }
 452
 453 int watchdog_ping(void) {
 454         usec_t ntime, timeout;
 455
 456         if (watchdog_timeout == 0)
 457                 return 0;
 458
 459         if (watchdog_fd < 0)
 460                 /* open_watchdog() will automatically ping the device for us if necessary */
 461                 return open_watchdog();
 462
 463         ntime = now(CLOCK_BOOTTIME);
 464         timeout = calc_timeout();
 465
 466         /* Never ping earlier than watchdog_timeout/4 and try to ping
 467          * by watchdog_timeout/2 plus scheduling latencies at the latest */
 468         if (timestamp_is_set(watchdog_last_ping)) {
 469                 assert(ntime >= watchdog_last_ping);
 470                 if ((ntime - watchdog_last_ping) < (timeout / 4))
 471                         return 0;
 472         }
 473
 474         return watchdog_ping_now();
 475 }
 476
 477 void watchdog_close(bool disarm) {
 478
 479         /* Once closed, pinging the device becomes a NOP and we request a new
 480          * call to watchdog_setup() to open the device again. */
 481         watchdog_timeout = 0;
 482
 483         if (watchdog_fd < 0)
 484                 return;
 485
 486         if (disarm) {
 487                 (void) watchdog_set_enable(false);
 488
 489                 /* To be sure, use magic close logic, too */
 490                 for (;;) {
 491                         static const char v = 'V';
 492
 493                         if (write(watchdog_fd, &v, 1) > 0)
 494                                 break;
 495
 496                         if (errno != EINTR) {
 497                                 log_warning_errno(errno, "Failed to disarm watchdog timer, ignoring: %m");
 498                                 break;
 499                         }
 500                 }
 501         }
 502
 503         watchdog_fd = safe_close(watchdog_fd);
 504 }