meson: allow WatchdogSec= in services to be configured

As discussed on systemd-devel [1], in Fedora we get lots of abrt reports
about the watchdog firing [2], but 100% of them seem to be caused by resource
starvation in the machine, and never actual deadlocks in the services being
monitored. Killing the services not only does not improve anything, but it
makes the resource starvation worse, because the service needs cycles to restart,
and coredump processing is also fairly expensive. This adds a configuration option
to allow the value to be changed. If the setting is not set, there is no change.

My plan is to set it to some ridiculusly high value, maybe 1h, to catch cases
where a service is actually hanging.

[1] https://lists.freedesktop.org/archives/systemd-devel/2019-October/043618.html
[2] https://bugzilla.redhat.com/show_bug.cgi?id=1300212

diff --git a/meson.build b/meson.build
index 1e27be78373c5bea63436618bfe38178be368d6c..dc1fde60ee3e08e35e045dc337b379a3c4dd3c71 100644 (file)
--- a/meson.build
+++ b/meson.build
@@ -795,6 +795,10 @@ conf.set_quoted('SYSTEMD_DEFAULT_LOCALE', default_locale)
 
 conf.set_quoted('GETTEXT_PACKAGE', meson.project_name())
 
+service_watchdog = get_option('service-watchdog')
+substs.set('SERVICE_WATCHDOG',
+           service_watchdog == '' ? '' : 'WatchdogSec=' + service_watchdog)
+
 substs.set('SUSHELL', get_option('debug-shell'))
 substs.set('DEBUGTTY', get_option('debug-tty'))
 conf.set_quoted('DEBUGTTY', get_option('debug-tty'))
@@ -3113,7 +3117,8 @@ status = [
         'default cgroup hierarchy:          @0@'.format(default_hierarchy),
         'default net.naming-scheme setting: @0@'.format(default_net_naming_scheme),
         'default KillUserProcesses setting: @0@'.format(kill_user_processes),
-        'default locale:                    @0@'.format(default_locale)]
+        'default locale:                    @0@'.format(default_locale),
+        'systemd service watchdog:          @0@'.format(service_watchdog == '' ? 'disabled' : service_watchdog)]
 
 alt_dns_servers = '\n                                            '.join(dns_servers.split(' '))
 alt_ntp_servers = '\n                                            '.join(ntp_servers.split(' '))

diff --git a/meson_options.txt b/meson_options.txt
index 5dc898eb804803a3f90f2ea20aff6e42ec8eb449..0919577fd763f3baf69cbe440650e178fb943780 100644 (file)
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -207,6 +207,8 @@ option('gshadow', type : 'boolean',
        description : 'support for shadow group')
 option('default-locale', type : 'string', value : '',
        description : 'default locale used when /etc/locale.conf does not exist')
+option('service-watchdog', type : 'string', value : '3min',
+       description : 'default watchdog setting for systemd services')
 
 option('default-dnssec', type : 'combo',
        description : 'default DNSSEC mode',

diff --git a/units/systemd-hostnamed.service.in b/units/systemd-hostnamed.service.in
index b4f606cf785a6b6e631b086a350ef4101e7509b0..1fbbafdd6f0ec8a63552fbb937701b29974323ef 100644 (file)
--- a/units/systemd-hostnamed.service.in
+++ b/units/systemd-hostnamed.service.in
@@ -36,4 +36,4 @@ RestrictSUIDSGID=yes
 SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service sethostname
-WatchdogSec=3min
+@SERVICE_WATCHDOG@

diff --git a/units/systemd-importd.service.in b/units/systemd-importd.service.in
index 38b7d7e94b4745f45e94f666fa101d8e3c76b9f9..1a6fae4b69daab0956828fa2298e564f5f9503ee 100644 (file)
--- a/units/systemd-importd.service.in
+++ b/units/systemd-importd.service.in
@@ -15,7 +15,6 @@ Documentation=https://www.freedesktop.org/wiki/Software/systemd/importd
 [Service]
 ExecStart=@rootlibexecdir@/systemd-importd
 BusName=org.freedesktop.import1
-WatchdogSec=3min
 KillMode=mixed
 CapabilityBoundingSet=CAP_CHOWN CAP_FOWNER CAP_FSETID CAP_MKNOD CAP_SETFCAP CAP_SYS_ADMIN CAP_SETPCAP CAP_DAC_OVERRIDE
 NoNewPrivileges=yes
@@ -28,3 +27,4 @@ SystemCallFilter=@system-service @mount
 SystemCallErrorNumber=EPERM
 SystemCallArchitectures=native
 LockPersonality=yes
+@SERVICE_WATCHDOG@

diff --git a/units/systemd-journal-remote.service.in b/units/systemd-journal-remote.service.in
index dd6322e62ccdae6853cd1641385870abe28f55d6..7f5238802ff933ca20280d683364f5a704e86032 100644 (file)
--- a/units/systemd-journal-remote.service.in
+++ b/units/systemd-journal-remote.service.in
@@ -33,7 +33,7 @@ RestrictRealtime=yes
 RestrictSUIDSGID=yes
 SystemCallArchitectures=native
 User=systemd-journal-remote
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 # If there are many split up journal files we need a lot of fds to access them
 # all in parallel.

diff --git a/units/systemd-journal-upload.service.in b/units/systemd-journal-upload.service.in
index e3800473ec5912ac1ade53e0cd93cf67c25df966..33ef3b8dcad4c79c04dc419cb1f543b5d67348bf 100644 (file)
--- a/units/systemd-journal-upload.service.in
+++ b/units/systemd-journal-upload.service.in
@@ -31,7 +31,7 @@ StateDirectory=systemd/journal-upload
 SupplementaryGroups=systemd-journal
 SystemCallArchitectures=native
 User=systemd-journal-upload
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 # If there are many split up journal files we need a lot of fds to access them
 # all in parallel.

diff --git a/units/systemd-journald.service.in b/units/systemd-journald.service.in
index 089bc38f5971260c3564f254a96fd3ca8007eda9..303d5a4826c11ebb6d8188018c9911ed5adea137 100644 (file)
--- a/units/systemd-journald.service.in
+++ b/units/systemd-journald.service.in
@@ -37,7 +37,7 @@ SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service
 Type=notify
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 # If there are many split up journal files we need a lot of fds to access them
 # all in parallel.

diff --git a/units/systemd-localed.service.in b/units/systemd-localed.service.in
index 7bca34409acec0fd8104f221d515203fa5fd1622..f9a81fa8ddd58058f08c0a92c8d34e89d9180a37 100644 (file)
--- a/units/systemd-localed.service.in
+++ b/units/systemd-localed.service.in
@@ -37,4 +37,4 @@ RestrictSUIDSGID=yes
 SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service
-WatchdogSec=3min
+@SERVICE_WATCHDOG@

diff --git a/units/systemd-logind.service.in b/units/systemd-logind.service.in
index c6f5b81c1d7641ad4640657e04803088a134a4cf..ef802a4e6f3629478fd22807b6a2d18288288c31 100644 (file)
--- a/units/systemd-logind.service.in
+++ b/units/systemd-logind.service.in
@@ -55,7 +55,7 @@ StateDirectory=systemd/linger
 SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 # Increase the default a bit in order to allow many simultaneous logins since
 # we keep one fd open per session.

diff --git a/units/systemd-machined.service.in b/units/systemd-machined.service.in
index d6deefea083a508ac458623a5443e00fefa4356d..3db0281f81dcb5f8589f5cd2c49b3cc4b93c6bc9 100644 (file)
--- a/units/systemd-machined.service.in
+++ b/units/systemd-machined.service.in
@@ -29,7 +29,7 @@ RestrictRealtime=yes
 SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service @mount
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 # Note that machined cannot be placed in a mount namespace, since it
 # needs access to the host's mount namespace in order to implement the

diff --git a/units/systemd-networkd.service.in b/units/systemd-networkd.service.in
index 5c6275e5b3043ee7be6e395fa449059f618ccfa7..ed985f64fa56729d30bc59e0593e645a3042395d 100644 (file)
--- a/units/systemd-networkd.service.in
+++ b/units/systemd-networkd.service.in
@@ -44,7 +44,7 @@ SystemCallFilter=@system-service
 Type=notify
 RestartKillSignal=SIGUSR2
 User=systemd-network
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 [Install]
 WantedBy=multi-user.target

diff --git a/units/systemd-nspawn@.service.in b/units/systemd-nspawn@.service.in
index 2473a730b472526e8d61373ed5e19748cfa0cb69..669fea3c12ca0f0b01cffb82e2d5c9e008e93828 100644 (file)
--- a/units/systemd-nspawn@.service.in
+++ b/units/systemd-nspawn@.service.in
@@ -23,10 +23,10 @@ KillMode=mixed
 Type=notify
 RestartForceExitStatus=133
 SuccessExitStatus=133
-WatchdogSec=3min
 Slice=machine.slice
 Delegate=yes
 TasksMax=16384
+@SERVICE_WATCHDOG@
 
 # Enforce a strict device policy, similar to the one nspawn configures when it
 # allocates its own scope unit. Make sure to keep these policies in sync if you

diff --git a/units/systemd-portabled.service.in b/units/systemd-portabled.service.in
index c88d3597b7a47ac3f40cbc12edad4d21e15914ac..fb79f454fd99593c06a2cdccd99e626939a346f4 100644 (file)
--- a/units/systemd-portabled.service.in
+++ b/units/systemd-portabled.service.in
@@ -15,7 +15,6 @@ RequiresMountsFor=/var/lib/portables
 [Service]
 ExecStart=@rootlibexecdir@/systemd-portabled
 BusName=org.freedesktop.portable1
-WatchdogSec=3min
 CapabilityBoundingSet=CAP_KILL CAP_SYS_PTRACE CAP_SYS_ADMIN CAP_SETGID CAP_SYS_CHROOT CAP_DAC_READ_SEARCH CAP_DAC_OVERRIDE CAP_CHOWN CAP_FOWNER CAP_FSETID CAP_MKNOD
 MemoryDenyWriteExecute=yes
 ProtectHostname=yes
@@ -26,3 +25,4 @@ SystemCallErrorNumber=EPERM
 SystemCallArchitectures=native
 LockPersonality=yes
 IPAddressDeny=any
+@SERVICE_WATCHDOG@

diff --git a/units/systemd-resolved.service.in b/units/systemd-resolved.service.in
index eee5d5ea8f4133fde2077cd6c55391a9c7c4c443..22cb20236379aa8ac3515ca2cf095c3a0a6f7fda 100644 (file)
--- a/units/systemd-resolved.service.in
+++ b/units/systemd-resolved.service.in
@@ -46,7 +46,7 @@ SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service
 Type=notify
 User=systemd-resolve
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 [Install]
 WantedBy=multi-user.target

diff --git a/units/systemd-timedated.service.in b/units/systemd-timedated.service.in
index d430ee201754a10195be466bc7b8e9f2c3b5a5d3..819cb4dba290bedd67906063d9c708b5f4b2917d 100644 (file)
--- a/units/systemd-timedated.service.in
+++ b/units/systemd-timedated.service.in
@@ -36,4 +36,4 @@ RestrictSUIDSGID=yes
 SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service @clock
-WatchdogSec=3min
+@SERVICE_WATCHDOG@

diff --git a/units/systemd-timesyncd.service.in b/units/systemd-timesyncd.service.in
index 2d8d14f6de0f5c5670bebc7c90237db798184ccc..1a866fcc7a8ea8e76f5db17ebe7545097d8dfd7e 100644 (file)
--- a/units/systemd-timesyncd.service.in
+++ b/units/systemd-timesyncd.service.in
@@ -46,7 +46,7 @@ SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service @clock
 Type=notify
 User=systemd-timesync
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 [Install]
 WantedBy=sysinit.target

diff --git a/units/systemd-udevd.service.in b/units/systemd-udevd.service.in
index c257af0efa5ad813fc88b332d438ecab1fbe6e78..8b1dd0efc73fa91877f167dd910fd29f67dd8c8e 100644 (file)
--- a/units/systemd-udevd.service.in
+++ b/units/systemd-udevd.service.in
@@ -25,7 +25,6 @@ RestartSec=0
 ExecStart=@rootlibexecdir@/systemd-udevd
 ExecReload=@rootbindir@/udevadm control --reload --timeout 0
 KillMode=mixed
-WatchdogSec=3min
 TasksMax=infinity
 PrivateMounts=yes
 ProtectHostname=yes
@@ -38,3 +37,4 @@ SystemCallErrorNumber=EPERM
 SystemCallArchitectures=native
 LockPersonality=yes
 IPAddressDeny=any
+@SERVICE_WATCHDOG@