Use LTP for testing valgrind

Add a new top level make target ltpchecks which will fetch the latest
linux test project (ltp) release as defined by the LTP_VERSION and
LTP_SHA256 variables in auxprogs/Makefile.am (update those when a new
version of ltp is released). If the ltp tar.xz has already been
downloaded, or it has already been unpacked and build, the (cached)
file and build will be reused.

The actual testing is done through the auxprogs/ltp-tester.sh script.
It takes all executable tests from the ltp testcases under
kernel/syscalls and runs them 3 times. Once directly, not under
valgrind, once with -q --tool=none and once with -q
--tool=memcheck. It then checks that valgrind didn't produce any
messages with the none tool, that there were no fatal errors produced
(as defined in auxprogs/ltp-error-patterns.txt) and that the ltp
results are the same with and without valgrind.

Currently there are 1472 test binaries and running them all (serially)
takes more than three hours and detects various missing or incomplete
syscall handlers in valgrind, plus various crashers.

https://bugs.kde.org/show_bug.cgi?id=502679

diff --git a/Makefile.am b/Makefile.am
index 14309dacfa8b6ffc9bd15022d5aeae49cf275069..e67356b5a0f2ca3db9e3113d5375e97f48db1703 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -103,6 +103,9 @@ perf: check
 auxchecks: all
        $(MAKE) -C auxprogs auxchecks
 
+ltpchecks: all
+       $(MAKE) -C auxprogs ltpchecks
+
 # Nb: no need to include any Makefile.am files here, or files included from
 # them, as automake includes them automatically.  Also not COPYING, README
 # or NEWS.

diff --git a/NEWS b/NEWS
index fac28e7984a8bec7d2308a7c2d2cdcaaa842b0a8..e77c62456b7c83d2100feccb9385d8b289e9ad24 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -76,6 +76,7 @@ are not entered into bugzilla tend to get forgotten about or ignored.
 502126  glibc 2.41 extra syscall_cancel frames
 502288  s390x: Memcheck false positives with NNPA last tensor dimension
 502324  s390x: Memcheck false positives with TMxx and TM/TMY
+502679  Use LTP for testing valgrind
 
 To see details of a given bug, visit
   https://bugs.kde.org/show_bug.cgi?id=XXXXXX

diff --git a/auxprogs/Makefile.am b/auxprogs/Makefile.am
index 3a9709da6f0633b94127edc28e7d6aa335d4870f..00c743bc11f8fad7997c80fbac85660f0b9c71db 100644 (file)
--- a/auxprogs/Makefile.am
+++ b/auxprogs/Makefile.am
@@ -124,6 +124,9 @@ endif
 auxchecks: gsl-check
 auxclean: gsl-clean
 
+ltpchecks: ltp-check
+ltpclean: ltp-clean
+
 # You can override AUX_CHECK_DIR to use a previous download/build.
 # Must be an absolute path.
 # e.g. make auxchecks AUX_CHECK_DIR=$HOME/valgrind-auxtests
@@ -148,6 +151,14 @@ GSL_CFLAGS+=-mfpmath=sse -msse2
 endif
 endif
 
+# Linux Test Project
+LTP_VERSION=20250130
+LTP_SHA256_SUM=02e4ec326be54c3fd92968229a468c02c665d168a8a673edc38a891f7395ae10
+LTP_TAR_NAME=ltp-full-$(LTP_VERSION).tar.xz
+LTP_URL=https://github.com/linux-test-project/ltp/releases/download/$(LTP_VERSION)/$(LTP_TAR_NAME)
+LTP_TAR=$(AUX_CHECK_DIR)/$(LTP_TAR_NAME)
+LTP_SRC_DIR=$(AUX_CHECK_DIR)/ltp-full-$(LTP_VERSION)
+
 # Trick to get a literal space to use in substitutions
 sp := $(subst ,, )
 
@@ -166,6 +177,10 @@ $(GSL_TAR):
        mkdir -p $(AUX_CHECK_DIR)
        wget -q -O $(GSL_TAR) $(GSL_URL)
 
+$(LTP_TAR):
+       mkdir -p $(AUX_CHECK_DIR)
+       wget -q -O $(LTP_TAR) $(LTP_URL)
+
 # We need to autoreconf to make sure to get config.guess, config.sub
 # and libtool for newer architectures.
 $(GSL_SRC_DIR)/gsl-patched: $(GSL_TAR)
@@ -177,6 +192,15 @@ $(GSL_SRC_DIR)/gsl-patched: $(GSL_TAR)
         autoreconf -f -i -Wnone)
        touch $@
 
+$(LTP_SRC_DIR): $(LTP_TAR)
+       echo "$(LTP_SHA256_SUM)  $(LTP_TAR)" | @SHA256SUM@ --check -
+       (cd $(AUX_CHECK_DIR) && \
+        tar Jxf $(LTP_TAR_NAME) && \
+        cd $(LTP_SRC_DIR) && \
+        ./configure CC="${CC}" CXX="${CXX}" CFLAGS="$(LTP_CFLAGS)" && \
+        ${MAKE} -j $(nproc) -C testcases/kernel/syscalls)
+       touch $@
+
 # We need make check -k because
 # some tests might fail even native (only on i386 though).
 # make check doesn't work reliably with -j.
@@ -204,17 +228,26 @@ gsl-check: $(GSL_BUILD_DIR)/gsl-build
        diff -u $(abs_top_srcdir)/auxprogs/gsl-1.6.out.x86.exp \
                $(GSL_BUILD_DIR)/valgrind-gsl.out
 
+ltp-check: $(LTP_SRC_DIR)
+       LTP_SRC_DIR=$(LTP_SRC_DIR) \
+       VALGRIND=$(abs_top_builddir)/vg-in-place \
+         $(abs_top_srcdir)/auxprogs/ltp-tester.sh
+
+
 # We keep the tarball but remove the unpacked sources and build
 gsl-clean:
        rm -rf $(GSL_SRC_NAME) $(GSL_BUILD_DIR)
 
+ltp-clean:
+       rm -rf $(LTP_SRC_DIR)
+
 #----------------------------------------------------------------------------
 # General stuff
 #----------------------------------------------------------------------------
 
 all-local: inplace-noinst_PROGRAMS inplace-noinst_DSYMS
 
-clean-local: clean-noinst_DSYMS auxclean
+clean-local: clean-noinst_DSYMS auxclean ltpclean
 
 install-exec-local: install-noinst_PROGRAMS install-noinst_DSYMS
 

diff --git a/auxprogs/ltp-error-patterns.txt b/auxprogs/ltp-error-patterns.txt
new file mode 100644 (file)
index 0000000..ee5048b
--- /dev/null
+++ b/auxprogs/ltp-error-patterns.txt
@@ -0,0 +1,6 @@
+VALGRIND INTERNAL ERROR
+^valgrind:
+WARNING: unhandled
+unhandled
+Valgrind does not support
+Valgrind abort

diff --git a/auxprogs/ltp-tester.sh b/auxprogs/ltp-tester.sh
new file mode 100755 (executable)
index 0000000..a3c2157
--- /dev/null
+++ b/auxprogs/ltp-tester.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -e
+
+if [ -z "${LTP_SRC_DIR:-}" ]; then
+    echo "ERROR: LTP_SRC_DIR needs to be set.  Dying, bye bye ..."
+    exit 1
+fi
+
+ORIG_PATH=$PATH
+ORIG_PWD=$PWD
+LOGDIR=${LOGDIR:-$LTP_SRC_DIR/valgrind-ltp-logs}
+SUMMARY_LOG="$LOGDIR/summary.log"
+DIFFCMD="diff -u"
+VALGRIND="${VALGRIND:-$LTP_SRC_DIR/../../../vg-in-place}"
+
+# Initialize LOGDIR
+mkdir -p $LOGDIR; rm -rf $LOGDIR/*
+
+myLog ()
+{
+    msg="$1"
+    echo "$msg"
+    echo -e "FAIL: $msg" >> $SUMMARY_LOG
+}
+
+cd $LTP_SRC_DIR
+
+mapfile -t files < <(find testcases/kernel/syscalls -executable -and -type f | sort)
+c=${#files[@]}; i=0
+
+for test in "${files[@]}"; do
+    dir=$(dirname $test)
+    exe=$(basename $test)
+    l="$LOGDIR/$exe"
+    mkdir -p $l
+    i=$((++i))
+    pushd $dir >/dev/null
+        echo "[$i/$c] Testing $exe ..." | tee -a $SUMMARY_LOG
+        PATH="$ORIG_PATH:$PWD"
+        ./$exe >$l/log1std 2>$l/log1err ||:
+        $VALGRIND -q --tool=none --log-file=$l/log2 ./$exe >$l/log2std 2>$l/log2err ||:
+        $VALGRIND -q --tool=memcheck --log-file=$l/log3 ./$exe >$l/log3std 2>$l/log3err ||:
+
+        # We want to make sure that LTP syscall tests give identical
+        # results with and without valgrind.  The test logs go to the
+        # stderr.  They aren't identical across individual runs. The
+        # differences include port numbers, temporary files, test
+        # output ordering changes and more. They aren't trivially
+        # comparable.  We resort to comparing at least the final
+        # summary of individual test results
+        tail -10 $l/log1err | grep -E "^(passed|failed|broken|skipped|warnings):" > $l/log1summary ||:
+        tail -10 $l/log2err | grep -E "^(passed|failed|broken|skipped|warnings):" > $l/log2summary ||:
+        tail -10 $l/log3err | grep -E "^(passed|failed|broken|skipped|warnings):" > $l/log3summary ||:
+
+        # Check logs, report errors
+        pushd $l >/dev/null
+            if test -s log2; then
+                myLog "${exe}: unempty log2:\n$(cat log2)"
+            fi
+
+            if grep -f $ORIG_PWD/ltp-error-patterns.txt * > error-patterns-found.txt; then
+                myLog "${exe}: error string found:\n$(cat error-patterns-found.txt)"
+            fi
+
+            if ! ${DIFFCMD} log1summary log2summary >/dev/null; then
+                myLog "${exe}: ${DIFFCMD} log1summary log2summary:\n$(${DIFFCMD} log1summary log2summary)"
+            fi
+
+            if ! ${DIFFCMD} log2summary log3summary >/dev/null; then
+                myLog "${exe}: ${DIFFCMD} log2summary log3summary:\n$(${DIFFCMD} log2summary log3summary)"
+            fi
+        popd >/dev/null
+    popd >/dev/null
+done
+
+echo "TESTING FINISHED, logs in $LOGDIR"
+