From: Martin Cermak Date: Thu, 17 Apr 2025 14:14:19 +0000 (+0200) Subject: Use LTP for testing valgrind X-Git-Tag: VALGRIND_3_25_0~26 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=944a8c864c5e1df6388bf4b47900eb9c001ceb73;p=thirdparty%2Fvalgrind.git Use LTP for testing valgrind Add a new top level make target ltpchecks which will fetch the latest linux test project (ltp) release as defined by the LTP_VERSION and LTP_SHA256 variables in auxprogs/Makefile.am (update those when a new version of ltp is released). If the ltp tar.xz has already been downloaded, or it has already been unpacked and build, the (cached) file and build will be reused. The actual testing is done through the auxprogs/ltp-tester.sh script. It takes all executable tests from the ltp testcases under kernel/syscalls and runs them 3 times. Once directly, not under valgrind, once with -q --tool=none and once with -q --tool=memcheck. It then checks that valgrind didn't produce any messages with the none tool, that there were no fatal errors produced (as defined in auxprogs/ltp-error-patterns.txt) and that the ltp results are the same with and without valgrind. Currently there are 1472 test binaries and running them all (serially) takes more than three hours and detects various missing or incomplete syscall handlers in valgrind, plus various crashers. https://bugs.kde.org/show_bug.cgi?id=502679 --- diff --git a/Makefile.am b/Makefile.am index 14309dacf..e67356b5a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -103,6 +103,9 @@ perf: check auxchecks: all $(MAKE) -C auxprogs auxchecks +ltpchecks: all + $(MAKE) -C auxprogs ltpchecks + # Nb: no need to include any Makefile.am files here, or files included from # them, as automake includes them automatically. Also not COPYING, README # or NEWS. diff --git a/NEWS b/NEWS index fac28e798..e77c62456 100644 --- a/NEWS +++ b/NEWS @@ -76,6 +76,7 @@ are not entered into bugzilla tend to get forgotten about or ignored. 502126 glibc 2.41 extra syscall_cancel frames 502288 s390x: Memcheck false positives with NNPA last tensor dimension 502324 s390x: Memcheck false positives with TMxx and TM/TMY +502679 Use LTP for testing valgrind To see details of a given bug, visit https://bugs.kde.org/show_bug.cgi?id=XXXXXX diff --git a/auxprogs/Makefile.am b/auxprogs/Makefile.am index 3a9709da6..00c743bc1 100644 --- a/auxprogs/Makefile.am +++ b/auxprogs/Makefile.am @@ -124,6 +124,9 @@ endif auxchecks: gsl-check auxclean: gsl-clean +ltpchecks: ltp-check +ltpclean: ltp-clean + # You can override AUX_CHECK_DIR to use a previous download/build. # Must be an absolute path. # e.g. make auxchecks AUX_CHECK_DIR=$HOME/valgrind-auxtests @@ -148,6 +151,14 @@ GSL_CFLAGS+=-mfpmath=sse -msse2 endif endif +# Linux Test Project +LTP_VERSION=20250130 +LTP_SHA256_SUM=02e4ec326be54c3fd92968229a468c02c665d168a8a673edc38a891f7395ae10 +LTP_TAR_NAME=ltp-full-$(LTP_VERSION).tar.xz +LTP_URL=https://github.com/linux-test-project/ltp/releases/download/$(LTP_VERSION)/$(LTP_TAR_NAME) +LTP_TAR=$(AUX_CHECK_DIR)/$(LTP_TAR_NAME) +LTP_SRC_DIR=$(AUX_CHECK_DIR)/ltp-full-$(LTP_VERSION) + # Trick to get a literal space to use in substitutions sp := $(subst ,, ) @@ -166,6 +177,10 @@ $(GSL_TAR): mkdir -p $(AUX_CHECK_DIR) wget -q -O $(GSL_TAR) $(GSL_URL) +$(LTP_TAR): + mkdir -p $(AUX_CHECK_DIR) + wget -q -O $(LTP_TAR) $(LTP_URL) + # We need to autoreconf to make sure to get config.guess, config.sub # and libtool for newer architectures. $(GSL_SRC_DIR)/gsl-patched: $(GSL_TAR) @@ -177,6 +192,15 @@ $(GSL_SRC_DIR)/gsl-patched: $(GSL_TAR) autoreconf -f -i -Wnone) touch $@ +$(LTP_SRC_DIR): $(LTP_TAR) + echo "$(LTP_SHA256_SUM) $(LTP_TAR)" | @SHA256SUM@ --check - + (cd $(AUX_CHECK_DIR) && \ + tar Jxf $(LTP_TAR_NAME) && \ + cd $(LTP_SRC_DIR) && \ + ./configure CC="${CC}" CXX="${CXX}" CFLAGS="$(LTP_CFLAGS)" && \ + ${MAKE} -j $(nproc) -C testcases/kernel/syscalls) + touch $@ + # We need make check -k because # some tests might fail even native (only on i386 though). # make check doesn't work reliably with -j. @@ -204,17 +228,26 @@ gsl-check: $(GSL_BUILD_DIR)/gsl-build diff -u $(abs_top_srcdir)/auxprogs/gsl-1.6.out.x86.exp \ $(GSL_BUILD_DIR)/valgrind-gsl.out +ltp-check: $(LTP_SRC_DIR) + LTP_SRC_DIR=$(LTP_SRC_DIR) \ + VALGRIND=$(abs_top_builddir)/vg-in-place \ + $(abs_top_srcdir)/auxprogs/ltp-tester.sh + + # We keep the tarball but remove the unpacked sources and build gsl-clean: rm -rf $(GSL_SRC_NAME) $(GSL_BUILD_DIR) +ltp-clean: + rm -rf $(LTP_SRC_DIR) + #---------------------------------------------------------------------------- # General stuff #---------------------------------------------------------------------------- all-local: inplace-noinst_PROGRAMS inplace-noinst_DSYMS -clean-local: clean-noinst_DSYMS auxclean +clean-local: clean-noinst_DSYMS auxclean ltpclean install-exec-local: install-noinst_PROGRAMS install-noinst_DSYMS diff --git a/auxprogs/ltp-error-patterns.txt b/auxprogs/ltp-error-patterns.txt new file mode 100644 index 000000000..ee5048bf4 --- /dev/null +++ b/auxprogs/ltp-error-patterns.txt @@ -0,0 +1,6 @@ +VALGRIND INTERNAL ERROR +^valgrind: +WARNING: unhandled +unhandled +Valgrind does not support +Valgrind abort diff --git a/auxprogs/ltp-tester.sh b/auxprogs/ltp-tester.sh new file mode 100755 index 000000000..a3c2157c5 --- /dev/null +++ b/auxprogs/ltp-tester.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +set -e + +if [ -z "${LTP_SRC_DIR:-}" ]; then + echo "ERROR: LTP_SRC_DIR needs to be set. Dying, bye bye ..." + exit 1 +fi + +ORIG_PATH=$PATH +ORIG_PWD=$PWD +LOGDIR=${LOGDIR:-$LTP_SRC_DIR/valgrind-ltp-logs} +SUMMARY_LOG="$LOGDIR/summary.log" +DIFFCMD="diff -u" +VALGRIND="${VALGRIND:-$LTP_SRC_DIR/../../../vg-in-place}" + +# Initialize LOGDIR +mkdir -p $LOGDIR; rm -rf $LOGDIR/* + +myLog () +{ + msg="$1" + echo "$msg" + echo -e "FAIL: $msg" >> $SUMMARY_LOG +} + +cd $LTP_SRC_DIR + +mapfile -t files < <(find testcases/kernel/syscalls -executable -and -type f | sort) +c=${#files[@]}; i=0 + +for test in "${files[@]}"; do + dir=$(dirname $test) + exe=$(basename $test) + l="$LOGDIR/$exe" + mkdir -p $l + i=$((++i)) + pushd $dir >/dev/null + echo "[$i/$c] Testing $exe ..." | tee -a $SUMMARY_LOG + PATH="$ORIG_PATH:$PWD" + ./$exe >$l/log1std 2>$l/log1err ||: + $VALGRIND -q --tool=none --log-file=$l/log2 ./$exe >$l/log2std 2>$l/log2err ||: + $VALGRIND -q --tool=memcheck --log-file=$l/log3 ./$exe >$l/log3std 2>$l/log3err ||: + + # We want to make sure that LTP syscall tests give identical + # results with and without valgrind. The test logs go to the + # stderr. They aren't identical across individual runs. The + # differences include port numbers, temporary files, test + # output ordering changes and more. They aren't trivially + # comparable. We resort to comparing at least the final + # summary of individual test results + tail -10 $l/log1err | grep -E "^(passed|failed|broken|skipped|warnings):" > $l/log1summary ||: + tail -10 $l/log2err | grep -E "^(passed|failed|broken|skipped|warnings):" > $l/log2summary ||: + tail -10 $l/log3err | grep -E "^(passed|failed|broken|skipped|warnings):" > $l/log3summary ||: + + # Check logs, report errors + pushd $l >/dev/null + if test -s log2; then + myLog "${exe}: unempty log2:\n$(cat log2)" + fi + + if grep -f $ORIG_PWD/ltp-error-patterns.txt * > error-patterns-found.txt; then + myLog "${exe}: error string found:\n$(cat error-patterns-found.txt)" + fi + + if ! ${DIFFCMD} log1summary log2summary >/dev/null; then + myLog "${exe}: ${DIFFCMD} log1summary log2summary:\n$(${DIFFCMD} log1summary log2summary)" + fi + + if ! ${DIFFCMD} log2summary log3summary >/dev/null; then + myLog "${exe}: ${DIFFCMD} log2summary log3summary:\n$(${DIFFCMD} log2summary log3summary)" + fi + popd >/dev/null + popd >/dev/null +done + +echo "TESTING FINISHED, logs in $LOGDIR" +