From: Martin Schwenke Date: Fri, 27 Feb 2026 07:20:21 +0000 (+1100) Subject: ctdb-scripts: Support periodic printing of warnings X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ce37002102ce5cc33ffeca519a631cccf62342aa;p=thirdparty%2Fsamba.git ctdb-scripts: Support periodic printing of warnings In the case where a failcount only produces warnings (not errors/unhealthy), the warnings could continue forever. A constant stream of warnings could dominate the logs, possibly obscuring other useful messages. To avoid this, add interval-based printing of warnings. It can be set to "hourly", "daily", any number of seconds (up to 4 digits) or "none" (no periodic warning, use current behaviour). The default is "hourly". Generally, the same interval will apply to all failcounts in a script. However, multiple carefully placed calls to failcount_set_warning_interval() can cause different intervals to be used for different counters. Add an extra unit test for vsftp monitoring to test this. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs --- diff --git a/ctdb/config/functions b/ctdb/config/functions index 99b2905d142..fd1b8c2db1d 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -965,8 +965,45 @@ _failcount_common() _thing="$1" _counter=$(echo "$_thing" | sed -e 's@/@_SLASH_@g' -e 's@ @_@g') + _failcount_warn_stamp="${script_state_dir}/${_counter}.last-warning" } +case "$CTDB_PLATFORM_STYLE" in +freebsd) + _stat_mtime_seconds() + { + stat -f '%m' -t '%s' "$@" 2>/dev/null + } + ;; +*) + _stat_mtime_seconds() + { + stat -c '%Y' "$@" 2>/dev/null + } + ;; +esac + +failcount_set_warning_interval() +{ + _interval="$1" + + case "$_interval" in + none) + failcount_interval="" + ;; + daily) + failcount_interval=$((24 * 60 * 60)) + ;; + hourly) + failcount_interval=$((60 * 60)) + ;; + [0-9] | [0-9][0-9] | [0-9][0-9][0-9] | [0-9][0-9][0-9][0-9]) + failcount_interval="$_interval" + ;; + esac +} +failcount_set_warning_interval "hourly" + failcount_init() { _thing="$1" @@ -989,6 +1026,7 @@ failcount_reset() printf 'NOTICE: %s: no longer failing\n' "$_thing" ctdb_counter_init "$_counter" + rm -f "$_failcount_warn_stamp" } failcount_incr() @@ -1039,12 +1077,32 @@ failcount_incr() fi fi + _print_output=false + # If warnings could be issued forever because this situation + # will never make the node unhealthy, then issue warnings + # according to failcount interval. In this case, also print + # the output. + if [ -z "$_unhealthy_threshold" ] && [ -n "$failcount_interval" ]; then + if _mtime=$(_stat_mtime_seconds "$_failcount_warn_stamp"); then + _now=$(date '+%s') + if [ $((_now - _mtime)) -lt "$failcount_interval" ]; then + return + fi + fi + _print_output=true + fi + + touch "$_failcount_warn_stamp" + printf 'WARNING: %s: fail count %d >= threshold %d\n' \ "$_thing" \ "$_failcount" \ "$_warn_threshold" if [ "$_failcount" -eq "$_warn_threshold" ] && [ -n "$_output" ]; then - # Only print output when reaching the warning threshold + # Also print output when reaching the warning threshold + _print_output=true + fi + if $_print_output; then echo "$_output" fi } diff --git a/ctdb/doc/ctdb-script.options.5.xml b/ctdb/doc/ctdb-script.options.5.xml index f2a7d34e1fd..988a8259905 100644 --- a/ctdb/doc/ctdb-script.options.5.xml +++ b/ctdb/doc/ctdb-script.options.5.xml @@ -144,10 +144,13 @@ A warning message is logged when failcount is - initially ≥ WARNING_THRESHOLD. No additional warnings - are logged (until failcount ≥ ERROR_THRESHOLD). The - intention is to avoid flooding the log with warnings. - The event script continues when a warning is logged. + initially ≥ WARNING_THRESHOLD. If ERROR_THRESHOLD is + provided then no additional warnings are logged (until + failcount ≥ ERROR_THRESHOLD). If ERROR_THRESHOLD is + not provided then additional warning are logged at + intervals of 1 hour. The intention is to avoid + flooding the log with warnings. The event script + continues when a warning is logged. diff --git a/ctdb/tests/UNIT/eventscripts/40.vsftpd.monitor.003.sh b/ctdb/tests/UNIT/eventscripts/40.vsftpd.monitor.003.sh new file mode 100755 index 00000000000..cd1c1f44289 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/40.vsftpd.monitor.003.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "periodic warnings, up once, down several times, back up, down again" + +setup "up" + +setup_script_options <= threshold 1 +vsftpd not listening on TCP port 21 +EOF +simple_test + +ok_null +simple_test + +setup_date_one_hour_from_now + +ok <= threshold 1 +vsftpd not listening on TCP port 21 +EOF +simple_test + +setup "up" + +ok <= threshold 1 +vsftpd not listening on TCP port 21 +EOF +simple_test + +ok_null +simple_test + +setup_date_one_hour_from_now + +ok <= threshold 1 +vsftpd not listening on TCP port 21 +EOF +simple_test + +setup "up" + +ok <