Add simulation tests

Use clknetsim to run multiple chronyd instances with simulated clocks
and network. It allows fast and reproducible testing, without real
network.

Included are several tests of performance in different clock/network
conditions, chronyd options, NTP authentication, chronyc, and past bug
fixes.

diff --git a/Makefile.in b/Makefile.in
index 58258af93c08c7b8e8b92ea132eb030f94aca8a4..962632529c66f3869ada4cdb38c9bb90b661483e 100644 (file)
--- a/Makefile.in
+++ b/Makefile.in
@@ -129,6 +129,9 @@ install: chronyd chronyc chrony.txt
 %.s : %.c
        $(CC) $(CFLAGS) $(CPPFLAGS) -S $<
 
+check :
+       cd test/simulation; ./run
+
 install-docs : docs
        [ -d $(DESTDIR)$(DOCDIR) ] || mkdir -p $(DESTDIR)$(DOCDIR)
        cp chrony.txt $(DESTDIR)$(DOCDIR)/chrony.txt

diff --git a/test/simulation/001-defaults b/test/simulation/001-defaults
new file mode 100755 (executable)
index 0000000..466d43b
--- /dev/null
+++ b/test/simulation/001-defaults
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+. test.common
+
+test_start "default test settings"
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_packet_interval || test_fail
+check_sync || test_fail
+
+test_pass

diff --git a/test/simulation/002-largenetwork b/test/simulation/002-largenetwork
new file mode 100755 (executable)
index 0000000..62885e5
--- /dev/null
+++ b/test/simulation/002-largenetwork
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+. test.common
+
+test_start "large network"
+
+time_rms_limit=5e-4
+
+server_strata=3
+servers=4
+clients=5
+
+client_start=2000
+min_sync_time=2100
+max_sync_time=2300
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_packet_interval || test_fail
+check_sync || test_fail
+
+test_pass

diff --git a/test/simulation/003-largefreqoffset b/test/simulation/003-largefreqoffset
new file mode 100755 (executable)
index 0000000..e463f0e
--- /dev/null
+++ b/test/simulation/003-largefreqoffset
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. test.common
+
+test_start "large frequency offset"
+
+max_sync_time=1000
+
+for freq_offset in -5e-2 -5e-3 5e-3 5e-2; do
+       # Adjust offset so it's close to 0 on first clock update
+       time_offset=$(awk "BEGIN {print -($freq_offset * 130)}")
+       run_test || test_fail
+       check_chronyd_exit || test_fail
+       check_source_selection || test_fail
+       check_sync || test_fail
+done
+
+test_pass

diff --git a/test/simulation/004-largetimeoffset b/test/simulation/004-largetimeoffset
new file mode 100755 (executable)
index 0000000..273056c
--- /dev/null
+++ b/test/simulation/004-largetimeoffset
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+. test.common
+
+test_start "large time offset"
+
+min_sync_time=1300
+max_sync_time=1400
+
+for time_offset in -1e2 1e2; do
+       run_test || test_fail
+       check_chronyd_exit || test_fail
+       check_source_selection || test_fail
+       check_sync || test_fail
+done
+
+test_pass

diff --git a/test/simulation/005-externalstep b/test/simulation/005-externalstep
new file mode 100755 (executable)
index 0000000..3f26324
--- /dev/null
+++ b/test/simulation/005-externalstep
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. test.common
+
+test_start "external time step"
+
+min_sync_time=1300
+max_sync_time=1500
+
+for step in -1e2 1e2; do
+       # Make one step in 150th second
+       client_step="(* $step (equal 0.1 (sum 1.0) 150))"
+       run_test || test_fail
+       check_chronyd_exit || test_fail
+       check_sync || test_fail
+done
+
+test_pass

diff --git a/test/simulation/006-largejitter b/test/simulation/006-largejitter
new file mode 100755 (executable)
index 0000000..50d5db9
--- /dev/null
+++ b/test/simulation/006-largejitter
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+. test.common
+
+test_start "large jitter"
+
+time_offset=1e0
+jitter=1e-1
+
+time_max_limit=5e-1
+freq_max_limit=2e-1
+time_rms_limit=1e-1
+freq_rms_limit=5e-3
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_packet_interval || test_fail
+check_sync || test_fail
+
+test_pass

diff --git a/test/simulation/007-largewander b/test/simulation/007-largewander
new file mode 100755 (executable)
index 0000000..22ffece
--- /dev/null
+++ b/test/simulation/007-largewander
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+. test.common
+
+test_start "large wander"
+
+wander=1e-7
+
+time_max_limit=5e-3
+freq_max_limit=5e-3
+time_rms_limit=1e-3
+freq_rms_limit=1e-4
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_packet_interval || test_fail
+check_sync || test_fail
+
+test_pass

diff --git a/test/simulation/101-poll b/test/simulation/101-poll
new file mode 100755 (executable)
index 0000000..ce9bd76
--- /dev/null
+++ b/test/simulation/101-poll
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+. test.common
+test_start "minpoll/maxpoll options"
+
+wander=0.0
+jitter=1e-6
+
+time_max_limit=1e-5
+freq_max_limit=1e-5
+time_rms_limit=5e-6
+freq_rms_limit=5e-6
+client_conf="makestep 1e-2 1"
+
+for poll in $(seq 2 14); do
+       client_server_options="minpoll $poll maxpoll $poll"
+       limit=$[2**$poll * 10]
+       min_sync_time=$[2**$poll * 2]
+       max_sync_time=$[2**$poll * 21 / 10 + 1]
+
+       run_test || test_fail
+       check_chronyd_exit || test_fail
+       check_source_selection || test_fail
+       check_packet_interval || test_fail
+       check_sync || test_fail
+done
+
+test_pass

diff --git a/test/simulation/102-iburst b/test/simulation/102-iburst
new file mode 100755 (executable)
index 0000000..dd66339
--- /dev/null
+++ b/test/simulation/102-iburst
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+. test.common
+test_start "iburst option"
+
+freq_offset=1e-4
+
+client_conf="makestep 1e-2 1
+driftfile tmp/drift"
+client_server_options="iburst"
+
+min_sync_time=4
+max_sync_time=6
+
+echo "100 1.0" > tmp/drift
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_packet_interval || test_fail
+check_sync || test_fail
+
+test_pass

diff --git a/test/simulation/103-initstepslew b/test/simulation/103-initstepslew
new file mode 100755 (executable)
index 0000000..89611f7
--- /dev/null
+++ b/test/simulation/103-initstepslew
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+. test.common
+test_start "initstepslew directive"
+
+freq_offset=0.0
+wander=0.0
+limit=100
+
+# clknetsim requires source port (if bound) to match dest port
+client_server_options="port 124"
+client_conf="initstepslew 5 192.168.123.1
+port 124"
+
+min_sync_time=15
+max_sync_time=30
+
+for time_offset in -2.0 -0.2 0.2 2.0; do
+       run_test || test_fail
+       check_chronyd_exit || test_fail
+       check_sync || test_fail
+done
+
+min_sync_time=1
+max_sync_time=1
+
+for time_offset in -1e8 -1e2 1e2 1e8; do
+       run_test || test_fail
+       check_sync || test_fail
+done
+
+test_pass

diff --git a/test/simulation/104-driftfile b/test/simulation/104-driftfile
new file mode 100755 (executable)
index 0000000..b65452d
--- /dev/null
+++ b/test/simulation/104-driftfile
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+. test.common
+test_start "driftfile directive"
+
+servers=0
+time_offset=0.0
+wander=0.0
+limit=10
+freq_max_limit=1e-9
+min_sync_time=1
+max_sync_time=1
+client_conf="driftfile tmp/drift"
+
+for freq_offset in -5e-2 -5e-4 -5e-6 5e-6 5e-4 5e-2; do
+       awk "BEGIN {printf \"%.9e 1\", 1e6 - 1 / (1 + $freq_offset) * 1e6}" > tmp/drift
+       run_test || test_fail
+       check_chronyd_exit || test_fail
+       check_sync || test_fail
+done
+
+test_pass

diff --git a/test/simulation/105-ntpauth b/test/simulation/105-ntpauth
new file mode 100755 (executable)
index 0000000..3e88d92
--- /dev/null
+++ b/test/simulation/105-ntpauth
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+. test.common
+
+test_start "NTP authentication"
+
+server_conf="keyfile tmp/keys"
+client_conf="keyfile tmp/keys"
+
+cat > tmp/keys <<-EOF
+1 $(tr -c -d 'a-zA-Z0-9' < /dev/urandom 2> /dev/null | head -c 24)
+2 ASCII:$(tr -c -d 'a-zA-Z0-9' < /dev/urandom 2> /dev/null | head -c 24)
+3 MD5 ASCII:$(tr -c -d 'a-zA-Z' < /dev/urandom 2> /dev/null | head -c 24)
+4 MD5 HEX:$(tr -c -d '0-9A-F' < /dev/urandom 2> /dev/null | head -c 32)
+EOF
+
+for key in 1 2 3 4; do
+       client_server_options="key $key"
+       run_test || test_fail
+       check_chronyd_exit || test_fail
+       check_source_selection || test_fail
+       check_packet_interval || test_fail
+       check_sync || test_fail
+done
+
+server_conf=""
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+# This check must fail as server doesn't know the key
+check_sync && test_fail
+
+test_pass

diff --git a/test/simulation/106-refclock b/test/simulation/106-refclock
new file mode 100755 (executable)
index 0000000..ed04406
--- /dev/null
+++ b/test/simulation/106-refclock
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. test.common
+test_start "SHM refclock"
+
+servers=0
+limit=1000
+refclock_jitter=$jitter
+min_sync_time=60
+max_sync_time=80
+client_conf="refclock SHM 0"
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_sync || test_fail
+
+test_pass

diff --git a/test/simulation/107-allowdeny b/test/simulation/107-allowdeny
new file mode 100755 (executable)
index 0000000..7b16312
--- /dev/null
+++ b/test/simulation/107-allowdeny
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+. test.common
+
+test_start "allow/deny directives"
+
+limit=500
+
+# Note that start_client in clknetsim.bash always adds allow to the config
+
+for server_conf in \
+       "deny" \
+       "deny all" \
+       "deny 192.168.0.0/16" \
+       "deny 192.168.123" \
+       "deny 192.168.123.2" \
+       "deny all
+allow 192.168.124.0/24"
+do
+       run_test || test_fail
+       check_chronyd_exit || test_fail
+       # These checks are expected to fail
+       check_source_selection && test_fail
+       check_packet_interval && test_fail
+       check_sync && test_fail
+done
+
+for server_conf in \
+       "deny all
+allow" \
+       "deny all
+allow all" \
+       "deny all
+allow 192.168.123" \
+       "deny all
+allow 192.168.123/24" \
+       "deny 192.168.124.0/24"
+do
+       run_test || test_fail
+       check_chronyd_exit || test_fail
+       check_source_selection || test_fail
+       check_packet_interval || test_fail
+       check_sync || test_fail
+done
+
+test_pass

diff --git a/test/simulation/108-peer b/test/simulation/108-peer
new file mode 100755 (executable)
index 0000000..b4d21d2
--- /dev/null
+++ b/test/simulation/108-peer
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+. test.common
+
+test_start "NTP peers"
+
+# Allow and drop packets to the server in 1000 second intervals, so only one
+# client has access to it and the other is forced to switch to the peer.
+base_delay=$(cat <<-EOF | tr -d '\n'
+  (+ 1e-4
+     (* -1
+        (equal 0.1 from 2)
+        (equal 0.1 to 1)
+        (equal 0.1 (min (% time 2000) 1000) 1000))
+     (* -1
+        (equal 0.1 from 3)
+        (equal 0.1 to 1)
+       (equal 0.1 (max (% time 2000) 1000) 1000)))
+EOF
+)
+
+clients=2
+peers=2
+max_sync_time=1000
+client_server_options="minpoll 6 maxpoll 6"
+client_peer_options="minpoll 6 maxpoll 6"
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_sync || test_fail
+
+test_pass

diff --git a/test/simulation/109-makestep b/test/simulation/109-makestep
new file mode 100755 (executable)
index 0000000..e74b719
--- /dev/null
+++ b/test/simulation/109-makestep
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+. test.common
+test_start "makestep directive"
+
+limit=200
+jitter=1e-5
+client_conf="makestep 2 1"
+
+min_sync_time=140
+max_sync_time=160
+
+for time_offset in -1.0 -0.1 0.1 1.0; do
+       run_test || test_fail
+       check_chronyd_exit || test_fail
+       check_source_selection || test_fail
+       check_sync || test_fail
+done
+
+min_sync_time=120
+max_sync_time=140
+
+for time_offset in -1e8 -1e2 1e2 1e8; do
+       run_test || test_fail
+       check_chronyd_exit || test_fail
+       check_source_selection || test_fail
+       check_sync || test_fail
+done
+
+test_pass

diff --git a/test/simulation/110-chronyc b/test/simulation/110-chronyc
new file mode 100755 (executable)
index 0000000..535a126
--- /dev/null
+++ b/test/simulation/110-chronyc
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+. test.common
+
+test_start "chronyc"
+
+chronyc_conf="tracking
+sources
+sourcestats"
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+
+check_chronyc_output "^Reference ID    : 192\.168\.123\.1 \(192\.168\.123\.1\)
+Stratum         : 2
+Ref time \(UTC\)  : Fri Jan  1 00:1.:.. 2010
+System time     : 0\.0000..... seconds (slow|fast) of NTP time
+Last offset     : -?0\.0000..... seconds
+RMS offset      : 0\.000...... seconds
+Frequency       : (99|100)\.... ppm fast
+Residual freq   : -?0.00. ppm
+Skew            : 0\.... ppm
+Root delay      : 0\.000... seconds
+Root dispersion : 0.000... seconds
+Update interval : 6.\.. seconds
+Leap status     : Normal
+210 Number of sources = 1
+MS Name/IP address         Stratum Poll Reach LastRx Last sample
+===============================================================================
+\^\* 192\.168\.123\.1                 1   [67]   377    [0-9]+ [0-9 +-]+[un]s\[[0-9 +-]+[un]s\] \+/-[ 0-9]+[un]s
+210 Number of sources = 1
+Name/IP Address            NP  NR  Span  Frequency  Freq Skew  Offset  Std Dev
+==============================================================================
+192\.168\.123\.1              [0-9 ]+ [0-9 ]+ [0-9 ]+ [ +-]0\.0..      0\....  [0-9 +-]+[un]s [0-9 ]+[un]s$" \
+|| test_fail
+
+test_pass

diff --git a/test/simulation/201-freqaccumulation b/test/simulation/201-freqaccumulation
new file mode 100755 (executable)
index 0000000..a8ba917
--- /dev/null
+++ b/test/simulation/201-freqaccumulation
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+. test.common
+
+# Test fix in commit 60d0fa299307076143da94d36deb7b908fa9bdb7
+
+test_start "frequency accumulation"
+
+time_offset=100.0
+jitter=1e-6
+base_delay=1e-6
+wander=0.0
+
+limit=200
+time_max_limit=1e-5
+freq_max_limit=1e-7
+time_rms_limit=1e-5
+freq_rms_limit=1e-7
+min_sync_time=120
+max_sync_time=140
+
+client_server_options="minpoll 6 maxpoll 6"
+client_conf="driftfile tmp/drift
+makestep 1 1"
+
+for freq_offset in -5e-2 -5e-4 5e-4 5e-2; do
+       for drift in -1e+4 -1e+2 1e+2 1e+4; do
+               echo "$drift 100000" > tmp/drift
+               run_test || test_fail
+               check_chronyd_exit || test_fail
+               check_sync || test_fail
+       done
+done
+
+test_pass

diff --git a/test/simulation/README b/test/simulation/README
new file mode 100644 (file)
index 0000000..c875ad6
--- /dev/null
+++ b/test/simulation/README
@@ -0,0 +1,12 @@
+This is a collection of simulation tests. They use clknetsim to simulate
+multiple systems connected in a network. It's available at
+
+https://github.com/mlichvar/clknetsim
+
+If this directory doesn't have a clknetsim subdirectory, a known working
+revision will be downloaded and compiled automatically.
+
+Currently it runs only on Linux.
+
+The tests are written in bash and they can be run directly. The ./run script
+runs all tests.

diff --git a/test/simulation/test.common b/test/simulation/test.common
new file mode 100644 (file)
index 0000000..6635128
--- /dev/null
+++ b/test/simulation/test.common
@@ -0,0 +1,413 @@
+# Copyright (C) 2013-2014  Miroslav Lichvar <mlichvar@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+export LC_ALL=C
+export PATH=../../:$PATH
+export CLKNETSIM_PATH=clknetsim
+
+# Known working clknetsim revision
+clknetsim_revision=eb0b85335c40027ac941b2276142542fff482107
+clknetsim_url=https://github.com/mlichvar/clknetsim/archive/$clknetsim_revision.tar.gz
+
+# Only Linux is supported
+if [ "$(uname -s)" != Linux ]; then
+       echo "Simulation tests supported only on Linux"
+       exit 3
+fi
+
+# Try to download clknetsim if not found
+if [ ! -e $CLKNETSIM_PATH ]; then
+       curl -L "$clknetsim_url" | tar xz || exit 3
+       ln -s clknetsim-$clknetsim_revision clknetsim || exit 3
+fi
+
+# Try to build clknetsim if not built
+if [ ! -x $CLKNETSIM_PATH/clknetsim -o ! -e $CLKNETSIM_PATH/clknetsim.so ]; then
+       make -C clknetsim || exit 3
+fi
+
+. $CLKNETSIM_PATH/clknetsim.bash
+
+# Default test testings
+
+default_limit=10000
+default_time_offset=1e-1
+default_freq_offset=1e-4
+default_base_delay=1e-4
+default_jitter=1e-4
+default_wander=1e-9
+default_refclock_jitter=""
+
+default_update_interval=0
+default_log_packets=0
+default_shift_pll=2
+
+default_server_strata=1
+default_servers=1
+default_clients=1
+default_peers=0
+default_server_start=0.0
+default_client_start=0.0
+default_chronyc_start=1000.0
+default_server_step=""
+default_client_step=""
+
+default_server_server_options=""
+default_client_server_options=""
+default_server_peer_options=""
+default_client_peer_options=""
+default_server_conf=""
+default_client_conf=""
+default_chronyc_conf=""
+default_chronyd_options=""
+
+default_time_max_limit=1e-3
+default_freq_max_limit=5e-4
+default_time_rms_limit=2e-4
+default_freq_rms_limit=1e-5
+default_min_sync_time=120
+default_max_sync_time=210
+
+# Initialize test settings from their defaults
+for defopt in $(declare | grep '^default_'); do
+       defoptname=${defopt%%=*}
+       optname=${defoptname#default_}
+       eval "[ -z \"\${$optname:+a}\" ] && $optname=\"\$$defoptname\""
+done
+
+test_start() {
+       rm -f tmp/*
+       echo "Testing $@:"
+}
+
+test_pass() {
+       echo "PASS"
+       exit 0
+}
+
+test_fail() {
+       echo "FAIL"
+       exit 1
+}
+
+test_ok() {
+       pad_line
+       echo -e "\tOK"
+       return 0
+}
+
+test_bad() {
+       pad_line
+       echo -e "\tBAD"
+       return 1
+}
+
+test_error() {
+       pad_line
+       echo -e "\tERROR"
+       return 1
+}
+
+msg_length=0
+pad_line() {
+       local line_length=56
+       [ $msg_length -lt $line_length ] && \
+               printf "%$[$line_length - $msg_length]s" ""
+       msg_length=0
+}
+
+# Print aligned message
+test_message() {
+       local level=$1 eol=$2
+       shift 2
+       local msg="$*"
+
+       while [ $level -gt 0 ]; do
+               echo -n "  "
+               level=$[$level - 1]
+               msg_length=$[$msg_length + 2]
+       done
+       echo -n "$msg"
+
+       msg_length=$[$msg_length + ${#msg}]
+       if [ $eol -ne 0 ]; then
+               echo
+               msg_length=0
+       fi
+}
+
+get_wander_expr() {
+       local scaled_wander
+
+       scaled_wander=$(awk "BEGIN {print $wander / \
+               sqrt($update_interval < 0 ? 2^-($update_interval) : 1)}")
+
+       echo "(+ $freq_offset (sum (* $scaled_wander (normal))))"
+}
+
+
+get_delay_expr() {
+       echo "(+ $base_delay (* $jitter (exponential)))"
+}
+
+get_refclock_expr() {
+       echo "(* $refclock_jitter (normal))"
+}
+
+get_chronyd_nodes() {
+       echo $[$servers * $server_strata + $clients]
+}
+
+get_chronyd_conf() {
+       local i stratum=$1 peer=$2
+
+       if [ $stratum -eq 1 ]; then
+               echo "local stratum 1"
+               echo "$server_conf"
+       elif [ $stratum -le $server_strata ]; then
+               for i in $(seq 1 $servers); do
+                       echo "server 192.168.123.$[$servers * ($stratum - 2) + $i] $server_server_options"
+               done
+               for i in $(seq 1 $peers); do
+                       [ $i -eq $peer -o $i -gt $servers ] && continue
+                       echo "peer 192.168.123.$[$servers * ($stratum - 1) + $i] $server_peer_options"
+               done
+               echo "$server_conf"
+       else
+               for i in $(seq 1 $servers); do
+                       echo "server 192.168.123.$[$servers * ($stratum - 2) + $i] $client_server_options"
+               done
+               for i in $(seq 1 $peers); do
+                       [ $i -eq $peer -o $i -gt $clients ] && continue
+                       echo "peer 192.168.123.$[$servers * ($stratum - 1) + $i] $client_peer_options"
+               done
+               echo "$client_conf"
+       fi
+}
+
+# Check if the clock was well synchronized
+check_sync() {
+       local i sync_time max_time_error max_freq_error ret=0
+       local rms_time_error rms_freq_error
+
+       test_message 2 1 "checking clock sync time, max/rms time/freq error:"
+
+       for i in $(seq 1 $(get_chronyd_nodes)); do
+               [ $i -gt $[$servers * $server_strata] ] || continue
+
+               sync_time=$(find_sync tmp/log.offset tmp/log.freq $i \
+                       $time_max_limit $freq_max_limit 1.0)
+               max_time_error=$(get_stat 'Maximum absolute offset' $i)
+               max_freq_error=$(get_stat 'Maximum absolute frequency' $i)
+               rms_time_error=$(get_stat 'RMS offset' $i)
+               rms_freq_error=$(get_stat 'RMS frequency' $i)
+
+               test_message 3 0 "node $i: $sync_time $(printf '%.2e %.2e %.2e %.2e' \
+                       $max_time_error $max_freq_error $rms_time_error $rms_freq_error)"
+
+               check_stat $sync_time $min_sync_time $max_sync_time && \
+                       check_stat $max_time_error 0.0 $time_max_limit && \
+                       check_stat $max_freq_error 0.0 $freq_max_limit && \
+                       check_stat $rms_time_error 0.0 $time_rms_limit && \
+                       check_stat $rms_freq_error 0.0 $freq_rms_limit && \
+                       test_ok || test_bad
+
+               [ $? -eq 0 ] || ret=1
+       done
+
+       return $ret
+}
+
+# Check if chronyd exited properly
+check_chronyd_exit() {
+       local i ret=0
+
+       test_message 2 1 "checking chronyd exit:"
+
+       for i in $(seq 1 $(get_chronyd_nodes)); do
+               test_message 3 0 "node $i:"
+
+               tail -n 1 tmp/log.$i | grep -q 'chronyd exiting' && \
+                       test_ok || test_bad
+               [ $? -eq 0 ] || ret=1
+       done
+
+       return $ret
+}
+
+# Check for problems in source selection
+check_source_selection() {
+       local i ret=0
+
+       test_message 2 1 "checking source selection:"
+
+       for i in $(seq $[$servers * $server_strata + 1] $(get_chronyd_nodes)); do
+               test_message 3 0 "node $i:"
+
+               ! grep -q 'no majority\|no reachable sources' tmp/log.$i && \
+                       grep -q 'Selected source' tmp/log.$i && \
+                       test_ok || test_bad
+               [ $? -eq 0 ] || ret=1
+       done
+
+       return $ret
+}
+
+# Check if incoming and outgoing packet intervals are sane
+check_packet_interval() {
+       local i ret=0 in_interval out_interval
+
+       test_message 2 1 "checking incoming and outgoing packet interval:"
+
+       for i in $(seq 1 $(get_chronyd_nodes)); do
+               in_interval=$(get_stat 'Mean incoming packet interval' $i)
+               out_interval=$(get_stat 'Mean outgoing packet interval' $i)
+
+               test_message 3 0 "node $i: $(printf '%.2e %.2e' \
+                       $in_interval $out_interval)"
+
+               # Check that the intervals are non-zero and shorter than limit,
+               # incoming is not longer than outgoing for stratum 1 servers, and
+               # outgoing is not longer than incoming for clients.
+               nodes=$[$servers * $server_strata + $clients]
+               check_stat $in_interval 0.1 $limit && \
+                       check_stat $out_interval 0.1 $limit && \
+                       ([ $i -gt $servers ] || \
+                               check_stat $in_interval 0.0 $out_interval) && \
+                       ([ $i -le $[$servers * $server_strata] ] || \
+                               check_stat $in_interval 0.0 $out_interval) && \
+                       test_ok || test_bad
+
+               [ $? -eq 0 ] || ret=1
+       done
+
+       return $ret
+}
+
+# Compare chronyc output with specified pattern
+check_chronyc_output() {
+       local i ret=0 pattern=$1
+
+       test_message 2 1 "checking chronyc output:"
+
+       for i in $(seq $[$(get_chronyd_nodes) + 1] $[$(get_chronyd_nodes) + $clients]); do
+               test_message 3 0 "node $i:"
+
+               [[ "$(cat tmp/log.$i)" =~ $pattern ]] && \
+                       test_ok || test_bad
+               [ $? -eq 0 ] || ret=1
+       done
+
+       return $ret
+}
+
+# Print test settings which differ from default value
+print_nondefaults() {
+       local defopt defoptname optname
+
+       test_message 2 1 "non-default settings:"
+       declare | grep '^default_*' | while read defopt; do
+               defoptname=${defopt%%=*}
+               optname=${defoptname#default_}
+               eval "[ \"\$$optname\" = \"\$$defoptname\" ]" || \
+                       test_message 3 1 $(eval "echo $optname=\$$optname")
+       done
+}
+
+run_simulation() {
+       local nodes=$1
+
+       test_message 2 0 "running simulation:"
+
+       start_server $nodes \
+               -o tmp/log.offset -f tmp/log.freq \
+               $([ $log_packets -ne 0 ] && echo -p tmp/log.packets) \
+               -R $(awk "BEGIN {print $update_interval < 0 ? 2^-($update_interval) : 1}") \
+               -r $(awk "BEGIN {print $max_sync_time * 2^$update_interval}") \
+               -l $(awk "BEGIN {print $limit * 2^$update_interval}") && test_ok || test_error
+}
+
+run_test() {
+       local i j n stratum node nodes step start freq offset conf
+
+       test_message 1 1 "network with $servers*$server_strata servers and $clients clients:"
+       print_nondefaults
+
+       nodes=$(get_chronyd_nodes)
+       [ -n "$chronyc_conf" ] && nodes=$[$nodes + $clients]
+
+       for i in $(seq 1 $nodes); do
+               echo "node${i}_shift_pll = $shift_pll"
+               for j in $(seq 1 $nodes); do
+                       [ $i -eq $j ] && continue
+                       echo "node${i}_delay${j} = $(get_delay_expr)"
+                       echo "node${j}_delay${i} = $(get_delay_expr)"
+               done
+       done > tmp/conf
+
+       node=1
+
+       for stratum in $(seq 1 $[$server_strata + 1]); do
+               [ $stratum -le $server_strata ] && n=$servers || n=$clients
+
+               for i in $(seq 1 $n); do
+                       test_message 2 0 "starting node $node:"
+                       if [ $stratum -eq 1 ]; then
+                               step=$server_step
+                               start=$server_start
+                               freq=""
+                               offset=0.0
+                       elif [ $stratum -le $server_strata ]; then
+                               step=$server_step
+                               start=$server_start
+                               freq=$(get_wander_expr)
+                               offset=0.0
+                       else
+                               step=$client_step
+                               start=$client_start
+                               freq=$(get_wander_expr)
+                               offset=$time_offset
+                       fi
+
+                       conf=$(get_chronyd_conf $stratum $i $n)
+
+                       [ -z "$freq" ] || echo "node${node}_freq = $freq" >> tmp/conf
+                       [ -z "$step" ] || echo "node${node}_step = $step" >> tmp/conf
+                       [ -z "$refclock_jitter" ] || \
+                               echo "node${node}_refclock = $(get_refclock_expr)" >> tmp/conf
+                       echo "node${node}_offset = $offset" >> tmp/conf
+                       echo "node${node}_start = $start" >> tmp/conf
+                       start_client $node chronyd "$conf" "" "$chronyd_options" && \
+                               test_ok || test_error
+
+                       [ $? -ne 0 ] && return 1
+                       node=$[$node + 1]
+               done
+       done
+
+       for i in $(seq 1 $[$nodes - $node + 1]); do
+               test_message 2 0 "starting node $node:"
+
+               echo "node${node}_start = $chronyc_start" >> tmp/conf
+               start_client $node chronyc "$chronyc_conf" "" \
+                       "-n -h 192.168.123.$[$node - $clients]" && \
+                       test_ok || test_error
+
+               [ $? -ne 0 ] && return 1
+               node=$[$node + 1]
+       done
+
+       run_simulation $nodes
+}