From: Martin Cermak Date: Fri, 16 May 2025 09:46:06 +0000 (+0200) Subject: PR503969: Make test results of make ltpchecks compatible with bunsen X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5894abc5fa9de30c6b4dde453bff3ac1034aa330;p=thirdparty%2Fvalgrind.git PR503969: Make test results of make ltpchecks compatible with bunsen Synthesize automake-like testcase log format for bunsen compatibility. Each testcase now produces a triplet of .{log,trs}, and a test-suite.log file. This way test results can be uploaded to bunsen using the t-upload-git-push command. The .log file contains the actual testcase log. The .trs file contains automake-encoded test result. The test-suite.log file contains testcase timing information. The log directory tree respect the one of the LTP testcases to avoid naming collisions. Here is how the test result was uploaded to Bunsen: > TESTING FINISHED, logs in /home/mcermak/WORK/valgrind/valgrind/auxprogs/auxchecks/ltp-full-20250130/ltp > make[1]: Leaving directory '/home/mcermak/WORK/valgrind/valgrind/auxprogs' > > real 8m27.467s > user 44m46.416s > sys 6m46.405s > 41$ cd /home/mcermak/WORK/valgrind/valgrind/auxprogs/auxchecks/ltp-full-20250130/ltp > 41$ t-upload-git-push ssh://builder.sourceware.org/git/bunsendb.git mcermak/PR503969-$(date +%s) \ > $(find . -type f -name '*.log' -o -name '*.trs' -o -name 'test-suite.log') > 3036eba2de1cc44a1942f9681720ea234da9029e refs/tags/mcermak/PR503969-1747389329 > 41$ Here's the resulting bunsen upload: https://builder.sourceware.org/testrun/3036eba2de1cc44a1942f9681720ea234da9029e --- diff --git a/NEWS b/NEWS index bae5f6fcf..d6fbbb41b 100644 --- a/NEWS +++ b/NEWS @@ -31,6 +31,7 @@ are not entered into bugzilla tend to get forgotten about or ignored. 504101 Add a "vgstack" script 504177 FILE DESCRIPTORS banner shows when closing some inherited fds 501741 syscall cachestat not wrapped +503969 Make test results of make ltpchecks compatible with bunsen 504265 FreeBSD: missing syscall wrappers for fchroot and setcred To see details of a given bug, visit diff --git a/auxprogs/ltp-tester.sh b/auxprogs/ltp-tester.sh index b5ba83c0b..036f196ce 100755 --- a/auxprogs/ltp-tester.sh +++ b/auxprogs/ltp-tester.sh @@ -8,24 +8,21 @@ if [ -z "${LTP_SRC_DIR:-}" ]; then fi ORIG_PATH=$PATH -ORIG_PWD=$PWD -LOGDIR=${LOGDIR:-$LTP_SRC_DIR/valgrind-ltp-logs} -SUMMARY_LOG="$LOGDIR/summary.log" +SCRIPT_SRC=$(dirname $0) +LOGDIR=${LOGDIR:-$LTP_SRC_DIR/ltp} DIFFCMD="diff -u" VALGRIND="${VALGRIND:-$LTP_SRC_DIR/../../../vg-in-place}" # For parallel testing, consider IO intensive jobs, take nproc into account PARALLEL_JOBS=${PARALLEL_JOBS:-$(nproc)} # TESTS env var may be specified to restrict testing to selected test cases +# Configure the LTP testsuite behavior per +# https://lore.kernel.org/ltp/20250505195003.GB137650@pevik/T/#t +export LTP_COLORIZE_OUTPUT=0 +export LTP_REPRODUCIBLE_OUTPUT=1 -# Initialize LOGDIR +# Initialize LOGDIR for bunsen upload (https://sourceware.org/bunsen/) mkdir -p $LOGDIR; rm -rf ${LOGDIR:?}/* -myLog () -{ - msg="$1" - echo "$msg" - echo -e "FAIL: $msg" >> summary -} doTest () { @@ -33,14 +30,17 @@ doTest () nr=$2 dir=$(dirname $t) exe=$(basename $t) - l="$LOGDIR/$exe" + l="$LOGDIR/$dir/$exe" + rv="PASS" mkdir -p $l - echo "[$nr/$c] Testing $exe ..." | tee -a $l/summary + echo "[$nr/$c] Testing $exe ..." pushd $dir >/dev/null PATH="$ORIG_PATH:$PWD" + t1=$(date +%s) ./$exe >$l/log1std 2>$l/log1err ||: $VALGRIND -q --tool=none --log-file=$l/log2 ./$exe >$l/log2std 2>$l/log2err ||: $VALGRIND -q --tool=memcheck --log-file=$l/log3 ./$exe >$l/log3std 2>$l/log3err ||: + t2=$(date +%s) # We want to make sure that LTP syscall tests give identical # results with and without valgrind. The test logs go to the @@ -56,20 +56,28 @@ doTest () # Check logs, report errors pushd $l >/dev/null if test -s log2; then - myLog "${exe}: unempty log2:\n$(cat log2)" + echo -e "${exe}: unempty log2:\n$(cat log2)" | tee -a $l/$exe.log + rv="FAIL" fi - if grep -f $ORIG_PWD/ltp-error-patterns.txt log* > error-patterns-found.txt; then - myLog "${exe}: error string found:\n$(cat error-patterns-found.txt)" + if grep -f $SCRIPT_SRC/ltp-error-patterns.txt log* > error-patterns-found.txt; then + echo -e "${exe}: error string found:\n$(cat error-patterns-found.txt)" | tee -a $l/$exe.log + rv="FAIL" fi if ! ${DIFFCMD} log1summary log2summary >/dev/null; then - myLog "${exe}: ${DIFFCMD} log1summary log2summary:\n$(${DIFFCMD} log1summary log2summary)" + echo -e "${exe}: ${DIFFCMD} log1summary log2summary:\n$(${DIFFCMD} log1summary log2summary)" | tee -a $l/$exe.log + rv="FAIL" fi if ! ${DIFFCMD} log2summary log3summary >/dev/null; then - myLog "${exe}: ${DIFFCMD} log2summary log3summary:\n$(${DIFFCMD} log2summary log3summary)" + echo -e "${exe}: ${DIFFCMD} log2summary log3summary:\n$(${DIFFCMD} log2summary log3summary)" | tee -a $l/$exe.log + rv="FAIL" fi + + # synthetize automake style testlogs for bunsen import + echo ":test-result: $rv" | tee -a $l/$exe.log > $l/$exe.trs + echo "Test time secs: $((t2 - t1))" > $l/test-suite.log popd >/dev/null popd >/dev/null } @@ -83,7 +91,7 @@ if [ -n "$TESTS" ]; then else echo "Running whole the LTP syscall testsuite ..." mapfile -t files < <(find testcases/kernel/syscalls -executable -and -type f \ - | sort | grep -v -f $ORIG_PWD/ltp-excludes.txt) + | sort | grep -v -f $SCRIPT_SRC/ltp-excludes.txt) fi c=${#files[@]}; i=0 @@ -97,11 +105,4 @@ done wait -# Reconstruct $SUMMARY_LOG -for test in "${files[@]}"; do - exe=$(basename $test) - l="$LOGDIR/$exe" - cat $l/summary >> $SUMMARY_LOG -done - echo "TESTING FINISHED, logs in $LOGDIR"