ctdb-tests: Add a new fetch ring test that also checks hot keys

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
Autobuild-User(master): Amitay Isaacs <amitay@samba.org>
Autobuild-Date(master): Fri May 22 08:05:54 UTC 2020 on sn-devel-184

diff --git a/ctdb/tests/INTEGRATION/database/fetch.002.ring-hotkeys.sh b/ctdb/tests/INTEGRATION/database/fetch.002.ring-hotkeys.sh
new file mode 100755 (executable)
index 0000000..c4e0023
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/database/fetch.002.ring-hotkeys.sh
@@ -0,0 +1,161 @@
+#!/bin/bash
+
+# Run the fetch_ring test, sanity check the output and check hot keys
+# statistics
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+testdb="fetch_ring.tdb"
+
+ctdb_get_all_pnns
+# $all_pnns is set above
+# shellcheck disable=SC2154
+num_nodes=$(echo "$all_pnns" | wc -w | tr -d '[:space:]')
+first=$(echo "$all_pnns" | sed -n -e '1p')
+
+get_key ()
+{
+       _n="$1"
+
+       echo "testkey${_n}"
+}
+
+run_fetch_ring ()
+{
+       _timelimit="$1"
+       _key_num="$2"
+
+       _key=$(get_key "$_key_num")
+       _base_cmd="fetch_ring -n ${num_nodes} -D ${testdb}"
+       _cmd="${_base_cmd} -t ${_timelimit} -k ${_key}"
+       echo "Running \"${_cmd}\" on all $num_nodes nodes."
+       testprog_onnode -v -p all "$_cmd"
+
+       _pat='^(Waiting for cluster|Fetch\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
+       sanity_check_output 1 "$_pat"
+
+       # Get the last line of output.
+       # $outfile is set above by testprog_onnode()
+       # shellcheck disable=SC2154
+       _last=$(tail -n 1 "$outfile")
+
+       # $last should look like this:
+       #    Fetch[1]: 10670.93 msgs/sec
+       _stuff="${_last##*Fetch\[*\]: }"
+       _mps="${_stuff% msgs/sec*}"
+
+       if [ "${_mps%.*}" -ge 10 ] ; then
+               echo "OK: ${_mps} msgs/sec >= 10 msgs/sec"
+       else
+               ctdb_test_fail "BAD: ${_mps} msgs/sec < 10 msgs/sec"
+       fi
+}
+
+check_hot_keys ()
+{
+       _pnn="$1"
+       _first_key="$2"
+       _num_keys="$3"
+
+       echo
+       echo "Checking hot keys on node ${_pnn}"
+
+       ctdb_onnode "$_pnn" dbstatistics "$testdb"
+
+       # Get hot keys with a non-empty key
+       _hotkeys=$(grep -Ex '[[:space:]]+Count:[[:digit:]]+ Key:[[:xdigit:]]+' \
+                       "$outfile") || true
+
+       # Check that there are the right number of non-empty slots
+       if [ -z "$_hotkeys" ] ; then
+               _num=0
+       else
+               _num=$(echo "$_hotkeys" | wc -l | tr -d '[:space:]')
+       fi
+       _msg="hot key slots in use = ${_num}"
+       if [ "$_num_keys" -ne "$_num" ] ; then
+               echo
+               cat "$outfile"
+               ctdb_test_fail "BAD: ${_msg} (expected ${_num_keys})"
+       fi
+       echo "GOOD: ${_msg}"
+
+       # No hot keys?  Done...
+       if [ "$_num" = 0 ] ; then
+               return
+       fi
+
+       # Check that hot key counts are correctly sorted
+       #
+       # Try to be as POSIX as possible
+       # shellcheck disable=SC2001
+       _counts=$(echo "$_hotkeys" | \
+                         sed -e 's|.*Count:\([[:digit:]][[:digit:]]*\).*|\1|')
+       _counts_sorted=$(echo "$_counts" | sort -n)
+       if [ "$_counts" != "$_counts_sorted" ] ; then
+               echo
+               cat "$outfile"
+               ctdb_test_fail "BAD: hot keys not sorted"
+       fi
+       echo "GOOD: hot key counts are correctly sorted"
+
+       # Check that all keys are considered hot
+       for _j in $(seq "$_first_key" $((_first_key + _num_keys - 1))) ; do
+               _key=$(get_key "$_j")
+               _key_hex=$(printf '%s' "$_key" | \
+                                  od -A n -t x1 | \
+                                  tr -d '[:space:]')
+               if ! echo "$_hotkeys" | grep -q "Key:${_key_hex}\$" ; then
+                       echo
+                       cat "$outfile"
+                       ctdb_test_fail "BAD: key \"${_key}\" is not a hot key"
+               fi
+       done
+       echo "GOOD: all keys are listed as hot keys"
+}
+
+# Run fetch_ring for each of 10 keys.  After each run confirm that all
+# keys used so far are considered hot keys (and do other hot key
+# sanity checks) on all nodes.
+for i in $(seq 1 10) ; do
+       run_fetch_ring 5 "$i"
+
+       for pnn in $all_pnns ; do
+               check_hot_keys "$pnn" 1 "$i"
+       done
+
+       echo
+done
+
+echo
+echo "Resetting statistics on node ${first}"
+ctdb_onnode "$first" statisticsreset
+
+# Ensure that only node $first has had statistics reset
+for pnn in $all_pnns ; do
+       if [ "$pnn" = "$first" ] ; then
+               check_hot_keys "$pnn" 1 0
+       else
+               check_hot_keys "$pnn" 1 10
+       fi
+done
+
+echo
+
+# Run fetch_ring for each of 3 new keys.  After each run confirm that
+# the new keys used so far are considered hot keys (and do other hot
+# key sanity checks) on node $first.
+#
+# Note that nothing can be said about hot keys on other nodes, since
+# they may be an arbitrary blend of old and new keys.
+for i in $(seq 1 3) ; do
+       run_fetch_ring 5 $((100 + i))
+
+       check_hot_keys 0 101 "$i"
+
+       echo
+done